def main(client):
# file_list = ['ableton__26_12_2019__12_33_10.html','argireline__26_12_2019__16_00_19.html']
for S3_REGION in S3_REGION_LIST:
print(S3_REGION)
output_dir = BATCH_OUTPUT_DIR_MOBILE.format(client)
INPUT_DIR = os.path.join(output_dir, S3_REGION, 'page_source')
OUTPUT_DIR = os.path.join(output_dir, S3_REGION,'showcase_result_computed')
if os.path.exists(INPUT_DIR):
create_dir(OUTPUT_DIR)
for file in os.listdir(INPUT_DIR):
# for file in file_list:
# print(file)
file_path = os.path.join(INPUT_DIR, file)
with open(file_path, 'r') as f:
page_data = f.read()
soup = BeautifulSoup(page_data,'html.parser')
query_clean_name = file.split('.')[0]
get_showcase_ad_details(OUTPUT_DIR, soup, query_clean_name)
else:
print('{} -- No showcase results for Mobile step 5.a'.format(S3_REGION))
print('--------------------------------------------')
def main(client):
# file_list = ['ableton__26_12_2019__26_12_2019__16_01_08.html','acqua_di_parma__26_12_2019__20_01_05.html','becca_highlighter__26_12_2019__20_01_07.html']
for S3_REGION in S3_REGION_LIST:
print(S3_REGION)
output_dir = BATCH_OUTPUT_DIR_MOBILE.format(client)
INPUT_DIR = os.path.join(output_dir, S3_REGION, 'page_source')
OUTPUT_DIR = os.path.join(output_dir, S3_REGION,
'sponsored_result_computed')
if os.path.exists(INPUT_DIR):
create_dir(OUTPUT_DIR)
for file in os.listdir(INPUT_DIR):
# for file in file_list:
# print(file)
file_path = os.path.join(INPUT_DIR, file)
with open(file_path, 'r') as f:
page_data = f.read()
soup = BeautifulSoup(page_data, 'html.parser')
query_clean_name = file.split('.')[0]
get_sponsored_ad_details(OUTPUT_DIR, soup,
query_clean_name)
else:
print('{} -- No sponsored results for Mobile step 4.a'.format(
S3_REGION))
print('--------------------------------------------')
def get_second_page_data(debug=False):
"""#get second page data, and save result to file ./api-data/second_page_data.json"""
t0 = time.time()
helper.remove_file(config.FILENAME_SECOND_PAGE_DATA)
helper.remove_file(config.FILENAME_SECOND_PAGE_SECTIONIDS_DATA)
helper.create_dir(config.SAVE_FILE_DIR)
helper.my_print("total has %d grades...\n" % len(config.GRADES), debug)
for grade in config.GRADES:
time.sleep(config.SLEEP_TIME)
helper.my_print("%s:" % grade, debug)
url = helper.get_second_page_url(grade=grade)
content = helper.get_url_content(url) # 获取知识章节列表
if content is None:
helper.my_print_error("the content is None.")
else:
helper.save_content_to_file(config.FILENAME_SECOND_PAGE_DATA,
content)
helper.my_print(content, debug)
# 获取知识章节对应的ID列表
json_data = json.loads(content, encoding='utf-8')
for course_list in json_data['list']:
for course in course_list['list']:
section_id = course['courseSectionID']
helper.save_content_to_file(
config.FILENAME_SECOND_PAGE_SECTIONIDS_DATA,
section_id)
helper.my_print(section_id, debug)
helper.my_print("", debug)
print('Done. %f seconds cost.' % (time.time() - t0))
def init():
# Initialize the output directory
h.create_dir(OUPTPUT_FOLDER_PATH)
# Initializie the workbook
workbook = Workbook(write_only=False)
worksheets = []
shipments = []
for i in xrange(SHIPMENTS):
worksheets.append(
Worksheet(workbook.create_sheet("Shipment%d" % (i + 1), i)))
shipments.append(Shipment(i))
return (workbook, worksheets, shipments)
def CopyFileToDestinationDir(self):
"""
Copies computed file to mounted s3 location on ec2
"""
src_file_path = os.path.join(
BASE_DIR, ADS_COMPUTED_OUTPUT_FILE_PATH.format(self.client))
client_output_dir = DESTINATION_ROOT_PATH.format(self.client)
create_dir(client_output_dir)
dest_file_path = os.path.join(client_output_dir,
ADS_COMPUTED_OUTPUT_FILE)
if os.path.exists(src_file_path):
copy_file(src_file_path, dest_file_path)
return dest_file_path
else:
return None
def run_tests(self):
self.learner.set_epsilon(0.0)
self.episode_dir = os.path.join(self.run_dir,
'episode_' + str(self.current_episode))
self.episode_dir = helper.create_dir(self.episode_dir)
self.test_steps = []
self.test_rewards = []
for test_pos in self.params['test_positions']:
self.init_episode()
self.run_episode(test_pos, tuple(self.current_task['goal_pos']),
self.current_task['name'])
self.test_steps.append(self.steps_in_episode)
self.test_rewards.append(self.reward_in_episode)
self.write_test_results()
self.learner.save_Qs(os.path.join(self.episode_dir, 'Qs.npy'))
# Make video from random position
if self.params['visual']:
self.set_status('recording')
self.init_episode()
self.run_episode(
self.env.get_random_state(tuple(
self.current_task['goal_pos'])),
tuple(self.current_task['goal_pos']),
self.current_task['name'])
self.learner.set_epsilon(self.learner.last_epsilon)
def get_main_page_data(debug=False):
"""#get main page data, and save the result to file ./api-data/main_page_data.json"""
helper.remove_file(config.FILENAME_MAIN_PAGE_DATA) # 先删除可能已存在的文件后, 再重新保存
helper.create_dir(config.SAVE_FILE_DIR)
t0 = time.time()
url = helper.get_main_page_url()
content = helper.get_url_content(url) # 获取首页的一级页面列表(包含年级列表,滚动图列表,学习计划列表等)
if content is None:
helper.my_print_error("main_page_data is None.")
else:
helper.save_content_to_file(config.FILENAME_MAIN_PAGE_DATA, content)
helper.my_print(content, debug)
print('Done. %f seconds cost.' % (time.time() - t0))
def get_software_download_dir(self):
"""
get software download directory to download OCP software bits
"""
default = '/home/ansible/files'
self.software_dir = '/home/ansible/files'
self.software_dir = set_values(self.software_dir, default)
dest_path_exist = check_path(self.software_dir, isdir=True)
if dest_path_exist:
logging.info('directory {} already exists'.format(self.software_dir))
else:
logging.info('Creating directory {}'.format(self.software_dir))
create_dir(self.software_dir)
self.inventory_dict['all']['vars']['software_src'] = self.software_dir
def get_third_page_data(debug=False):
"""#get third page data, and save result to file ./api-data/third_page_data.json"""
t0 = time.time()
helper.remove_file(config.FILENAME_THIRD_PAGE_DATA)
helper.remove_file(config.FILENAME_THIRD_SECTIONIDS_DATA)
helper.create_dir(config.SAVE_FILE_DIR)
with open(config.FILENAME_SECOND_PAGE_SECTIONIDS_DATA) as f:
i = 0
lines = f.readlines()
helper.my_print("total has %d chapters...\n" % len(lines), debug)
for line in lines:
i += 1
section_id = str(int(line))
helper.my_print("line:%d sectionID:%s" % (i, section_id), debug)
time.sleep(config.SLEEP_TIME)
url = helper.get_third_page_url(section_id)
content = helper.get_url_content(
url) # 根据某个章节的 sectionID, 获取其知识点列表
if content is None:
helper.my_print_error("the content is None.")
else:
helper.save_content_to_file(config.FILENAME_THIRD_PAGE_DATA,
content)
helper.my_print(content, debug)
# 获取知识点对应的课程ID列表(用于根据课程ID, 获取题目列表)
json_data = json.loads(content)
for course in json_data['list']:
course_dic = {
'courseSectionID': course['courseSectionID'],
'sectionName': course['sectionName'],
'parentID': course['parentID']
}
data = json.dumps(course_dic, ensure_ascii=False)
helper.save_content_to_file(
config.FILENAME_THIRD_SECTIONIDS_DATA, data)
helper.my_print(data, debug)
helper.my_print("", debug)
print('Done. %f seconds cost.' % (time.time() - t0))
def clone_repo(self, repo, repo_dir):
"""
Clones the repo passed as an argument into repos_dir directory. Note that this method clones a basic version of
the repo (depth=1, branch=master)
:param repo: repository to be cloned
:param repo_dir: directory where repository will be cloned
:return:
"""
project_dir = os.path.join(repo_dir, repo['project'])
helper.create_dir(project_dir)
clone_url = repo['cloneUrl'].split('@')[0] + ':' + urllib.quote(self.credentials['password'],
safe='') + '@' + \
repo['cloneUrl'].split('@')[1]
command = "git clone --depth 1 -b master %s repos/%s/%s" % (
pipes.quote(clone_url), pipes.quote(
repo['project']), pipes.quote(repo['slug']))
os.system(command)
def get_software_download_dir(self):
"""
get software download directory to download OCP 4.3 software bits
"""
self.clear_screen()
default = '/home/ansible/files'
self.software_dir = input('provide complete path of directory to download OCP 4.3 software bits\n'
'default [/home/ansible/files]: ')
self.software_dir = set_values(self.software_dir, default)
dest_path_exist = check_path(self.software_dir, isdir=True)
if dest_path_exist:
logging.info('directory {} already exists'.format(self.software_dir))
else:
logging.info('Creating directory {}'.format(self.software_dir))
create_dir(self.software_dir)
self.inventory_dict['csah']['vars']['software_src'] = self.software_dir
def init_run(self):
_logger.info("..... Starting run %s" % str(self.current_run))
run_dir = os.path.join(self.task_dir, 'run_' + str(self.current_run))
self.run_dir = helper.create_dir(run_dir)
# Create run stats file: run_stats.csv
self.run_stats_file = os.path.join(self.run_dir, 'stats_run.csv')
self.run_steps = 0
helper.write_stats_file(self.run_stats_file, 'episode', 'steps_total',
'steps_mean', 'reward_total', 'reward_mean',
'epsilon', 'step_count')
self._init_run()
def run(self):
global S3_BUCKET_DICT
for S3_REGION in S3_REGION_LIST:
BUCKET = S3_BUCKET_DICT[S3_REGION]
OUTPUT_DIR = os.path.join(self.batch_output_dir, S3_REGION)
create_dir(OUTPUT_DIR)
# For thread safety
session = boto3.session.Session()
if SERVER:
s3 = session.resource('s3')
else:
s3 = session.resource(
's3',
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
print('{}: {}'.format(self.thread_name, BUCKET))
bucket = s3.Bucket(BUCKET)
files = bucket.objects.filter(Prefix=S3_PREFIX_PC)
for file in files:
file_key = file.key
if '_mobile/' not in file_key and CURRENT_DATE_STAMP in file_key:
if 'sponsored_result' in file_key and self.client_name in file_key:
# print('{}: {}'.format(self.thread_name,file_key))
# print(file_key)
file_dir = file_key.split('/')[-2]
file_path = os.path.join(OUTPUT_DIR, file_dir)
create_dir(file_path)
file_name = file_key.split('/')[-1]
download_file_path = os.path.join(file_path, file_name)
if not os.path.exists(download_file_path):
obj = s3.Object(
BUCKET,
file_key).download_file(download_file_path)
print('{} Completed'.format(self.thread_name))
def preprocess(self):
"""
Naive preprocessor that creates the dataset for CLAMS by cloning all repos and filtering locally.
:return:
"""
repos_dir = os.path.join(os.getcwd(), 'repos')
helper.create_dir(repos_dir)
bitbucket_client = BitBucketServerClient(
host=self.bitbucket_host,
is_ssh=False,
credentials=self.bitbucket_credentials)
repos = bitbucket_client.get_bitbucket_server_repos(self.client_repos)
self.clone_repos(bitbucket_client, repos, repos_dir)
for project in projects_map:
package_name = projects_map[project]['package']
print "Removing previous session's results..."
directory = os.path.join(os.getcwd(), 'files', project)
helper.delete_dir(directory)
helper.create_dir(directory)
print "Ready to run new session!\n"
# use the following command to filter files
os.system("find ./repos -iname '*.java' | xargs -n16 -P8 grep -l" +
" \"" + package_name + "\" > " + project + ".txt")
fname = project + ".txt"
files_urls = {}
if os.path.exists(fname):
self.process_filtered_results(directory, files_urls, fname,
project)
print 'Writing files\' BitBucket Server urls to file...'
helper.write_json(files_urls, 'files_urls', directory)
print 'Files\' BitBucket urls are now stored in a json file!\n'
else:
print 'No usage examples found for ' + project
def preprocess(self):
"""
This method runs the preprocessor that creates the dataset for CLAMS by using Hound to filter down to specific files.
:return:
"""
for project in projects_map:
package_name = projects_map[project]['package']
print "Removing previous session's results..."
directory = os.path.join(os.getcwd(), 'files', project)
helper.delete_dir(directory)
helper.create_dir(directory)
print "Ready to run new session!\n"
# search on Hound
print "\nSearching on Hound..."
hound_client = HoundClient(self.hound_host, self.hound_credentials)
hound_query = {'q': 'import ' + package_name, 'i': 'nope', 'files': '.java', 'repos': '*'}
json_response = hound_client.search(hound_query)
files_urls = self.parse_hound_response(project, json_response)
print "Search completed!\n"
# download files from BitBucket Server
print "Downloading files..."
bitbucket_client = BitBucketServerClient(host=self.bitbucket_host, is_ssh=self.is_bitbucket_ssh,
credentials=self.bitbucket_credentials)
for file_name, info in files_urls.iteritems():
response = bitbucket_client.download_file(info)
helper.write_file_content(response, file_name, directory, self.is_bitbucket_ssh)
print "Files are now stored locally!\n"
print 'Writing files\' BitBucket Server urls to file...'
helper.write_json(files_urls, 'files_urls', directory)
print 'Files\' BitBucket Server urls are now stored in a json file!\n'
# sleep for 1s to avoid overloading Hound/BitBucket
# remove in case you don't have any latency issues
time.sleep(1)
def init_task(self):
_logger.info("##### Starting task %s" % str(self.current_task['name']))
task_dir = os.path.join(self.exp_dir,
'task_' + self.current_task['name'])
self.task_dir = helper.create_dir(task_dir)
self._init_task()
def get_question_data(debug=False, start=1, count=0):
"""#get question data, and save result to file ./api-data/question_data.json"""
t0 = time.time()
with open(config.FILENAME_THIRD_SECTIONIDS_DATA) as f:
i = 0
lines = f.readlines()
line_count = len(lines)
# 检查开始索引参数 start
try:
start_index = int(start)
except ValueError as e:
helper.my_print_error(
'Error: %s\n the "start" param must be a Integer number!' % e)
return
else:
if start_index < 1 or start_index > line_count:
helper.my_print_error(
'Error: the "start" param must in range(1, len(lines)+1)')
return
# 检查获取条目参数 count
try:
limit_count = int(count)
except ValueError as e:
helper.my_print_error(
'Error: %s\n the "count" param must be a Integer number!' % e)
return
else:
if limit_count <= 0 or (start_index - 1 +
limit_count) > line_count:
limit_count = line_count - start_index + 1
if start_index == 1: # 当 start_index 值为1时, 表示重新开始获取, 此时应先清除之间存在的数据文件
helper.remove_file(config.FILENAME_QUESTION_DATA)
helper.create_dir(config.SAVE_FILE_DIR)
helper.my_print("total has %d sections...\n" % line_count, debug)
for line in lines:
i += 1
if i < start_index:
continue
elif i >= (start_index + limit_count):
break
json_data = json.loads(line)
section_id = json_data['courseSectionID']
helper.my_print("line:%d sectionID:%s" % (i, section_id), debug)
time.sleep(config.SLEEP_TIME)
url = helper.get_question_url(section_id)
content = helper.get_url_content(url) # 根据知识点的 sectionID, 获取题目对表
if content is None: # 若发生了异常时, 直接退出
helper.my_print_error("the content is None.")
helper.my_print_error(
"An Exception has happen. get_question_data will be exit. "
"the next start_index should be %d" % i)
break
else:
result_data = json.loads(content)
if result_data['code'] != '99999': # 若返回码不是表示"成功"时, 直接退出
helper.my_print_error(
"result code is not 99999. get_question_data will be exit. "
"the next start_index should be %d" % i)
break
helper.save_content_to_file(config.FILENAME_QUESTION_DATA,
content)
helper.my_print(content, debug)
helper.my_print("", debug)
print('Done. %f seconds cost.' % (time.time() - t0))
def main(all_datasets, gpu, epoch):
device = torch.device('cuda:{}'.format(gpu))
attack_names = ['DeepFool', 'CarliniWagner']
for attack_name in attack_names:
csv_rows = []
for dataset in all_datasets:
ds_obj, datasets, data_loaders = \
hp.get_data_loder_objects(dataset, PHASES, **hp.get_loader_kwargs(batch_size))
for dir_name in os.listdir('../{}/adversarial_images/'.format(
ds_obj.name)):
# dir_name contains model name and other params, process them here
if 'RegularizedLoss' in dir_name:
model_name = dir_name.split('_RegularizedLoss_')[0]
with_regularization = True
tau = float(dir_name.split('_tau_')[1].split('_')[0])
alpha = float(dir_name.split('_alpha_')[1].split('_')[0])
if 'probabilities' in dir_name:
probabilities = True
else:
probabilities = False
if 'exact' in dir_name:
sigmoid_approx = False
else:
sigmoid_approx = True
else:
model_name = dir_name
with_regularization = False
tau, alpha, sigmoid_approx, probabilities = None, None, None, None
if 'robust' in dir_name:
robust_regularization = True
beta = float(dir_name.split('_beta_')[1].split('_')[0])
gamma = float(dir_name.split('_gamma_')[1].split('_')[0])
else:
robust_regularization = False
beta, gamma = None, None
# for model_name in DATASET_TO_MODEL_NAMES[dataset.split('_')[0].lower()]:
# taus = DATASET_TO_MODEL_TO_TAUS[dataset.split('_')[0].lower()][model_name]
# alphas = DATASET_TO_MODEL_TO_ALPHAS[dataset.split('_')[0].lower()][model_name]
# for (tau_idx, tau), (alpha_idx, alpha) in itertools.product(*[enumerate(taus), enumerate(alphas)]):
regularization_params = {
'tau': tau,
'alpha': alpha,
'sigmoid_approx': sigmoid_approx,
'probabilities': probabilities,
'robust_regularization': robust_regularization,
'beta': beta,
'gamma': gamma,
'device': device
}
model_to_load = model.DNN(
model_name=model_name,
num_classes=ds_obj.num_classes(),
learning_rate=learning_rate,
aggregate_coeff=aggregate_coeff,
with_regularization=with_regularization,
regularization_params=regularization_params)
complete_model_name = '{}_{}'.format(model_to_load.model_name, model_to_load.criterion._get_name()) \
if not isinstance(model_to_load.criterion, nn.CrossEntropyLoss) else model_to_load.model_name
print('Attack: {}, Dataset: {}, Model: {}'.format(
attack_name, dataset, complete_model_name))
adv_folder = '../{}/adversarial_images/{}/{}'.format(
ds_obj.name, complete_model_name, attack_name)
adv_image_ids, all_adv_objs = hp.load_adversarial_objects(
folder=adv_folder,
epoch=epoch,
ds_obj=ds_obj,
device=device)
all_images_adversarial = [x.image for x in all_adv_objs]
print(adv_folder)
print(
len(glob.glob("{}/*_epoch_{}*".format(adv_folder, epoch))))
if 'cifar' in ds_obj.name.lower():
if ds_obj.name.lower() == 'cifar10':
sensitive_attrs, sensitive_attrs_names = [], []
for cname in ds_obj.classes:
sensitive_attrs_names.append(cname)
sensitive_attrs.append(np.array([1 if ds_obj.classes[ds_obj.test_labels[int(img_id)]] == cname \
else 0 for img_id in adv_image_ids]))
else:
sensitive_attrs = [np.array(
[1 if ds_obj.classes[ds_obj.test_labels[int(img_id)]] == ds_obj.name.split('_')[-1].lower() \
else 0 for img_id in adv_image_ids])]
sensitive_attrs_names = [
ds_obj.name.lower().split('_')[-1]
]
else:
attr = ds_obj.name.lower().split('_')[-1]
sensitive_attrs = [np.array([ds_obj.get_image_protected_class('test', int(img_id), attr=attr) \
for img_id in adv_image_ids])] # sens_attr = 1 means minority
sensitive_attrs_names = [
'Black' if attr == 'race' else 'Female'
]
majority_differences, minority_differences = [], []
for sensitive_attr in sensitive_attrs:
minority_difference, majority_difference = image_differences(
adv_image_ids, all_images_adversarial, sensitive_attr,
ds_obj)
majority_differences.append(majority_difference)
minority_differences.append(minority_difference)
for minority_difference, majority_difference, sensitive_attr_name in zip(
minority_differences, majority_differences,
sensitive_attrs_names):
mu_minority, mu_majority = np.mean(
minority_difference), np.mean(majority_difference)
csv_rows.append([
dataset, complete_model_name, sensitive_attr_name,
mu_minority, mu_majority
])
hp.create_dir("pickled_ubs")
df = pd.DataFrame(
csv_rows,
columns=['dataset', 'model', 'minority', 'mu_min', 'mu_maj'])
df.to_csv('pickled_ubs/{}_cdf_mus_regularized.csv'.format(attack_name),
index=False)
print('Saved to pickled_ubs/{}_cdf_mus_regularized.csv!'.format(
attack_name))
def analyse_results(SM_PC,
SM_BC,
RM_PC,
RM_BC,
selection,
StateM_PC,
StateM_BC,
seed,
multiplier,
linear,
pklf_name,
dir_name,
analyse_replay=True,
TFR=True,
save=True,
verbose=True):
"""
Analyses results from simulations (see `detect_oscillations.py`)
:param SM_PC, SM_BC, RM_PC, RM_BC: Brian2 spike and rate monitors of PC and BC populations (see `run_simulation()`)
:param selection: array of selected cells used by PC multi state monitor
:param seed: random seed used to run the sim - here used only for saving
:param multiplier: weight matrix multiplier (see `spw_network_wmx_mult.py`)
:param linear: bool for linear/circular weight matrix (more advanced replay detection is used in linear case)
:param pklf_name: file name of saved place fileds used for replay detection in the linear case
:param dir_name: subdirectory name used to save replay detection (and optionally TFR) figures in linear case
:param analyse_replay: bool for analysing replay or not
:param TFR: bool for calculating time freq. repr. (using wavelet analysis) or not
:param save: bool for saving results
:param verbose: bool for printing results or not
"""
if SM_PC.num_spikes > 0 and SM_BC.num_spikes > 0: # check if there is any activity
spike_times_PC, spiking_neurons_PC, rate_PC, ISI_hist_PC, bin_edges_PC = preprocess_monitors(
SM_PC, RM_PC)
spike_times_BC, spiking_neurons_BC, rate_BC = preprocess_monitors(
SM_BC, RM_BC, calc_ISI=False)
if not linear:
plot_raster(spike_times_PC,
spiking_neurons_PC,
rate_PC, [ISI_hist_PC, bin_edges_PC],
None,
"blue",
multiplier_=multiplier)
subset = plot_zoomed(spike_times_PC,
spiking_neurons_PC,
rate_PC,
"PC_population",
"blue",
multiplier_=multiplier,
StateM=StateM_PC,
selection=selection)
plot_zoomed(spike_times_BC,
spiking_neurons_BC,
rate_BC,
"BC_population",
"green",
multiplier_=multiplier,
PC_pop=False,
StateM=StateM_BC)
plot_detailed(StateM_PC, subset, multiplier_=multiplier)
if not linear:
slice_idx = []
replay_ROI = np.where((150 <= bin_edges_PC)
& (bin_edges_PC <= 850))
replay = replay_circular(
ISI_hist_PC[replay_ROI]) if analyse_replay else np.nan
else:
slice_idx = slice_high_activity(rate_PC, th=2, min_len=260)
plot_raster(spike_times_PC,
spiking_neurons_PC,
rate_PC, [ISI_hist_PC, bin_edges_PC],
slice_idx,
"blue",
multiplier_=multiplier)
if analyse_replay:
replay, replay_results = replay_linear(spike_times_PC,
spiking_neurons_PC,
slice_idx,
pklf_name,
N=30)
else:
replay, replay_results = np.nan, {}
if slice_idx and replay_results != {}:
create_dir(dir_name)
for bounds, tmp in replay_results.items():
fig_name = os.path.join(
dir_name, "%i-%i_replay.png" % (bounds[0], bounds[1]))
plot_posterior_trajectory(tmp["X_posterior"],
tmp["fitted_path"], tmp["R"],
fig_name)
if save:
save_replay_analysis(replay, replay_results, seed)
mean_rate_PC, rate_ac_PC, max_ac_PC, t_max_ac_PC, f_PC, Pxx_PC = analyse_rate(
rate_PC, 1000., slice_idx)
mean_rate_BC, rate_ac_BC, max_ac_BC, t_max_ac_BC, f_BC, Pxx_BC = analyse_rate(
rate_BC, 1000., slice_idx)
plot_PSD(rate_PC,
rate_ac_PC,
f_PC,
Pxx_PC,
"PC_population",
"blue",
multiplier_=multiplier)
plot_PSD(rate_BC,
rate_ac_BC,
f_BC,
Pxx_BC,
"BC_population",
"green",
multiplier_=multiplier)
t_LFP, LFP, f_LFP, Pxx_LFP = analyse_estimated_LFP(
StateM_PC, selection, slice_idx)
plot_LFP(t_LFP, LFP, f_LFP, Pxx_LFP, multiplier_=multiplier)
if save:
save_LFP(t_LFP, LFP, seed)
save_PSD(f_PC, Pxx_PC, f_BC, Pxx_BC, f_LFP, Pxx_LFP, seed)
if TFR:
coefs_PC, freqs_PC = calc_TFR(rate_PC, 1000., slice_idx)
coefs_BC, freqs_BC = calc_TFR(rate_BC, 1000., slice_idx)
coefs_LFP, freqs_LFP = calc_TFR(LFP[::10].copy(), 1000., slice_idx)
if not linear or not slice_idx:
plot_TFR(
coefs_PC, freqs_PC, "PC_population",
os.path.join(base_path, "figures",
"%.2f_PC_population_wt.png" % multiplier))
plot_TFR(
coefs_BC, freqs_BC, "BC_population",
os.path.join(base_path, "figures",
"%.2f_BC_population_wt.png" % multiplier))
plot_TFR(
coefs_LFP, freqs_LFP, "LFP",
os.path.join(base_path, "figures",
"%.2f_LFP_wt.png" % multiplier))
else:
for i, bounds in enumerate(slice_idx):
fig_name = os.path.join(
dir_name,
"%i-%i_PC_population_wt.png" % (bounds[0], bounds[1]))
plot_TFR(coefs_PC[i], freqs_PC, "PC_population", fig_name)
fig_name = os.path.join(
dir_name,
"%i-%i_BC_population_wt.png" % (bounds[0], bounds[1]))
plot_TFR(coefs_BC[i], freqs_PC, "BC_population", fig_name)
fig_name = os.path.join(
dir_name, "%i-%i_LFP_wt.png" % (bounds[0], bounds[1]))
plot_TFR(coefs_LFP[i], freqs_LFP, "LFP", fig_name)
if save:
save_TFR(freqs_PC, coefs_PC, freqs_BC, coefs_BC, freqs_LFP,
coefs_LFP, seed)
max_ac_ripple_PC, t_max_ac_ripple_PC = ripple_AC(rate_ac_PC, slice_idx)
max_ac_ripple_BC, t_max_ac_ripple_BC = ripple_AC(rate_ac_BC, slice_idx)
avg_ripple_freq_PC, ripple_power_PC = ripple(f_PC, Pxx_PC, slice_idx)
avg_ripple_freq_BC, ripple_power_BC = ripple(f_BC, Pxx_BC, slice_idx)
avg_ripple_freq_LFP, ripple_power_LFP = ripple(f_LFP, Pxx_LFP,
slice_idx)
avg_gamma_freq_PC, gamma_power_PC = gamma(f_PC, Pxx_PC, slice_idx)
avg_gamma_freq_BC, gamma_power_BC = gamma(f_BC, Pxx_BC, slice_idx)
avg_gamma_freq_LFP, gamma_power_LFP = gamma(f_LFP, Pxx_LFP, slice_idx)
if verbose:
if not np.isnan(replay):
print("Replay detected!")
else:
print("No replay detected...")
print("Mean excitatory rate: %.3f" % mean_rate_PC)
print("Mean inhibitory rate: %.3f" % mean_rate_BC)
print("Average exc. ripple freq: %.3f" % avg_ripple_freq_PC)
print("Exc. ripple power: %.3f" % ripple_power_PC)
print("Average inh. ripple freq: %.3f" % avg_ripple_freq_BC)
print("Inh. ripple power: %.3f" % ripple_power_BC)
print("Average LFP ripple freq: %.3f" % avg_ripple_freq_LFP)
print("LFP ripple power: %.3f" % ripple_power_LFP)
return [
multiplier, replay, mean_rate_PC, mean_rate_BC, avg_ripple_freq_PC,
ripple_power_PC, avg_ripple_freq_BC, ripple_power_BC,
avg_ripple_freq_LFP, ripple_power_LFP, avg_gamma_freq_PC,
gamma_power_PC, avg_gamma_freq_BC, gamma_power_BC,
avg_gamma_freq_LFP, gamma_power_LFP, max_ac_PC, max_ac_ripple_PC,
max_ac_BC, max_ac_ripple_BC
]
else:
if verbose:
print("No activity!")
return [np.nan for _ in range(20)]
def main(dataset,
gpu,
model_name,
epochs,
taus,
alphas,
with_regularization=False,
sigmoid_approx=False,
probabilities=False,
robust_regularization=False,
betas=[None],
gammas=[None]):
device = torch.device('cuda:{}'.format(gpu))
attack_names = ['DeepFool', 'CarliniWagner']
ds_obj, datasets, data_loaders = \
hp.get_data_loder_objects(dataset, PHASES, **hp.get_loader_kwargs(batch_size))
for epoch in epochs:
for (tau_idx, tau), (alpha_idx, alpha), (beta_idx, beta), (gamma_idx, gamma) in \
itertools.product(*[enumerate(taus), enumerate(alphas), enumerate(betas), enumerate(gammas)]):
regularization_params = {
'tau': tau,
'alpha': alpha,
'sigmoid_approx': sigmoid_approx,
'probabilities': probabilities,
'robust_regularization': robust_regularization,
'beta': beta,
'gamma': gamma,
'device': device
}
model_to_load = model.DNN(
model_name=model_name,
num_classes=ds_obj.num_classes(),
learning_rate=learning_rate,
aggregate_coeff=aggregate_coeff,
with_regularization=with_regularization,
regularization_params=regularization_params)
# filename = '{}_{}_epoch_{}_lr_{}.pth'.format(model_to_load.model_name, model_to_load.criterion._get_name(),
# epoch, learning_rate)
# model_to_load.model_ft.load_state_dict(torch.load('../{}/model_weights/{}'.format(ds_obj.name, filename),
# map_location=device))
# model_to_load.model_ft.eval()
# print ('Loaded weights from: ../{}/model_weights/{}'.format(ds_obj.name, filename))
complete_model_name = '{}_{}'.format(model_to_load.model_name, model_to_load.criterion._get_name()) \
if not isinstance(model_to_load.criterion, nn.CrossEntropyLoss) else model_to_load.model_name
for attack_name in attack_names:
adv_folder = '../{}/adversarial_images/{}/{}'.format(
ds_obj.name, complete_model_name, attack_name)
adv_image_ids, all_adv_objs = hp.load_adversarial_objects(
folder=adv_folder,
epoch=epoch,
ds_obj=ds_obj,
device=device)
all_images_adversarial = [x.image for x in all_adv_objs]
print(adv_folder)
print(
len(glob.glob("{}/*_epoch_{}*".format(adv_folder, epoch))))
if 'cifar' in ds_obj.name.lower():
if ds_obj.name.lower() == 'cifar10':
sensitive_attrs, sensitive_attrs_names = [], []
for cname in ds_obj.classes:
sensitive_attrs_names.append(cname)
sensitive_attrs.append(np.array([1 if ds_obj.classes[ds_obj.test_labels[int(img_id)]] == cname \
else 0 for img_id in adv_image_ids]))
else:
sensitive_attrs = [np.array(
[1 if ds_obj.classes[ds_obj.test_labels[int(img_id)]] == ds_obj.name.split('_')[-1].lower() \
else 0 for img_id in adv_image_ids])]
sensitive_attrs_names = [
ds_obj.name.lower().split('_')[-1]
]
else:
attr = ds_obj.name.lower().split('_')[-1]
sensitive_attrs = [np.array([ds_obj.get_image_protected_class('test', int(img_id), attr=attr) \
for img_id in adv_image_ids])]
sensitive_attrs_names = [
'Black' if attr == 'race' else 'Female'
]
majority_differences, minority_differences = [], []
for sensitive_attr in sensitive_attrs:
minority_difference, majority_difference = image_differences(
adv_image_ids, all_images_adversarial, sensitive_attr,
ds_obj)
majority_differences.append(majority_difference)
minority_differences.append(minority_difference)
# print (minority_difference, majority_difference)
hp.create_dir("plots/{}".format(ds_obj.name))
hp.create_dir("plots/{}/{}".format(ds_obj.name,
model_to_load.model_name))
hp.create_dir("plots/{}/{}/{}".format(ds_obj.name,
model_to_load.model_name,
attack_name))
dir_to_save = "plots/{}/{}/{}".format(ds_obj.name,
model_to_load.model_name,
attack_name)
# taus = np.linspace(0.0, 0.5, 2000)
taus = np.linspace(0.0, 2.0, 2000)
# taus = np.linspace(0.0, 2.0, 2000) if 'deepfool' in attack_name.lower() else np.linspace(2.9, 3.1, 2000)
for minority_difference, majority_difference, sensitive_attr_name in zip(
minority_differences, majority_differences,
sensitive_attrs_names):
frac_greater_than_tau_majority = np.array([
np.sum(majority_difference > t) /
len(majority_difference) for t in taus
])
frac_greater_than_tau_minority = np.array([
np.sum(minority_difference > t) /
len(minority_difference) for t in taus
])
if paper_friendly_plots:
set_paper_friendly_plots_params()
fig = plt.figure()
if not paper_friendly_plots:
fig.suptitle(
r'fraction $d_\theta > \tau$ for {}'.format(
ds_obj.name),
fontsize=20)
ax = fig.add_subplot(111)
ax.plot(taus,
frac_greater_than_tau_majority,
color='blue',
label='Other Classes')
ax.plot(taus,
frac_greater_than_tau_minority,
color='red',
label='{}'.format(sensitive_attr_name))
ax.set_xlabel('Distance to Adv. Sample' + r' ($\tau$)')
ax.set_ylabel(r'$ \widehat{I^\tau_s} $')
plt.legend()
extension = 'png' if not paper_friendly_plots else 'pdf'
filename = '{}_inv_cdf'.format(model_to_load.criterion._get_name()) \
if not isinstance(model_to_load.criterion, nn.CrossEntropyLoss) else \
'inv_cdf_{}'.format(sensitive_attr_name)
plt.savefig('{}/{}.{}'.format(dir_to_save, filename,
extension),
bbox_inches='tight')
plt.show()
plt.close()
def set_exp_dir(self):
folder = "%s_%s_%s" % (str(
time.strftime("%Y-%m-%d_%H-%M")), str(
self.params['type']).lower(), str(self.params['grid']).lower())
path_to_dir = os.path.join(os.getcwd(), 'logs', folder)
return helper.create_dir(path_to_dir)
def calibrate(self):
"""Returns Intrinsic and Extrinsic properties of camera.
Intrinsic properties of camera: focal length (fx, fy) and center (Cx, Cy)
Extrinsic camera properties: distortion coefficients, rotation vector and translation vector
"""
assert os.path.exists(self._source_dir), \
'"{}" must exist and contain calibration images.'.format(self._source_dir)
for _, fname in tqdm(enumerate(os.listdir(self._source_dir)), total=self.num, desc='Loading chessboard images'):
img = cv2.imread(os.path.join(self._source_dir, fname), cv2.IMREAD_GRAYSCALE)
_, mask = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
self.masks.append(mask)
for _, img in tqdm(enumerate(self.masks), total=self.num, desc='Detecting chessboard images'):
ret, corners = cv2.findChessboardCorners(img, self.chessboard_size, cv2.CALIB_CB_ADAPTIVE_THRESH +
cv2.CALIB_CB_FAST_CHECK + cv2.CALIB_CB_NORMALIZE_IMAGE)
# If found, add object point, image points after refining them
if ret:
self.objpts.append(self.objpt)
# refine corner location based on criteria
self.corners_ = cv2.cornerSubPix(img, corners, (11, 11), (-1, -1), self.criteria)
self.imgpts.append(self.corners_)
else:
raise ChessBoardError()
print(f'\nCalibrating {self.num} valid chessboards images found {self._source_dir} ...\n')
self.dims = img.shape[::-1]
self.h, self.w = self.dims
self.img_orig = img.copy()
# Draw and display the corners
cv2.drawChessboardCorners(self.img_orig, self.chessboard_size, self.corners_, ret)
# Calibrate camera
ret, self.cam_mtx, self.dist_coeff, self.rvecs, self.tvecs = cv2.calibrateCamera(self.objpts, self.imgpts,
self.dims, None, None)
create_dir(self.target_dir)
# Mean reprojection error
mean_err = 0
for i in range(len(self.objpts)):
r_mtx_, _ = cv2.Rodrigues(self.rvecs[i], jacobian=0)
imgpts_, _ = cv2.projectPoints(self.objpts[i], self.rvecs[i], self.tvecs[i], self.cam_mtx, self.dist_coeff)
error = cv2.norm(self.imgpts[i], imgpts_, cv2.NORM_L2) / len(imgpts_)
mean_err += error
self.errors.append(error)
self.rot_mtx.append(r_mtx_)
self.mean_err = mean_err / len(self.objpts)
if self.show:
plt.imshow(self.img_orig, cmap='hot')
plt.axis('off')
plt.savefig(f'{self.target_dir}/{self.filename}_calibrated.png', dpi=1000)
pos = np.arange(self.num)
plt.figure()
plt.bar(pos, self.errors, width=0.6, color='blue')
plt.axhline(self.mean_err, color='red', linestyle='--')
plt.ylabel('Mean error in Pixels')
plt.xlabel('Image')
plt.savefig(f'{self.target_dir}/{self.filename}_mean_error.png', dpi=1000)
plt.show()
else:
plt.imshow(self.img_orig, cmap='hot')
plt.axis('off')
plt.savefig(f'{self.target_dir}/{self.filename}_calibrated.png', dpi=1000)
pos = np.arange(self.num)
plt.figure()
plt.bar(pos, self.errors, width=0.6, color='blue')
plt.axhline(self.mean_err, color='red', linestyle='--')
plt.ylabel('Mean error in Pixels')
plt.xlabel('Image')
plt.savefig(f'{self.target_dir}/{self.filename}_mean_error.png', dpi=1000)
self.save_calibration_results()
