def insert_table(self, _filename, tmp_jsonfile, table, columns_index,strict_str=None, header=None, sep='\t'):
fd=open(_filename)
line_count=len(fd.readlines())
print(line_count)
widgets = ["processing file: "+_filename, progressbar.Percentage(), ' ', progressbar.Bar(marker=progressbar.RotatingMarker()),' ', progressbar.ETA(), ' ', progressbar.FileTransferSpeed()]
pbar = progressbar.ProgressBar(widgets=widgets, maxval=line_count).start()
fd=open(_filename)
if header==None:
header=fd.readline().replace("\n",'').split(sep)
i=0
batch=fd.readlines(2097152)
while batch!=[]:
fdw=open(tmp_jsonfile,'w')
batch_array=[]
for line in batch:
pbar.update(i+1)
i=i+1
aline=line.replace("\n",'').split(sep)
reg={}
for i,field in enumerate(aline):
if i+1 in columns_index:
if i+1 in strict_str:
reg[header[i]]=field
else:
try:
reg[header[i]]=int(field)
except:
try:
reg[header[i]]=float(field)
except:
reg[header[i]]=field
batch_array.append(reg)
json.dump(batch_array, fdw)
fdw.close()
self.insert_doc(table,tmp_jsonfile)
batch=fd.readlines(2097152)
pbar.finish()
def __process_zones(self, zones, user, show_progressbar=False):
errors = []
items = []
widget = [
progressbar.FormatLabel(''), ' ',
progressbar.Percentage(), ' ',
progressbar.Bar('#'), ' ',
progressbar.RotatingMarker(), ' ',
progressbar.ETA()
]
if show_progressbar:
widget[0] = progressbar.FormatLabel('Processing zones')
bar = progressbar.ProgressBar(max_value=len(zones), widgets=widget)
domain_mapping = self.__get_domain_mapping(user.id)
user_base_domain = '.' + self.__dns_zones.get_base_domain(
user.admin, user.username)
count = 0
for zone in zones:
count += 1
bar.update(count) if show_progressbar else False
active = True if zone['active'] in ['1', 'yes', 'true'] else False
catch_all = True if zone['catch_all'] in ['1', 'yes', 'true'
] else False
forwarding = True if zone['forwarding'] in ['1', 'yes', 'true'
] else False
regex = True if zone['regex'] in ['1', 'yes', 'true'] else False
master = True if zone['master'] in ['1', 'yes', 'true'] else False
tags = zone['tags'].split(',')
# Trim each element.
map(str.strip, tags)
# Remove empty elements.
tags = list(filter(None, tags))
is_valid = True
if not user.admin:
if zone['domain'][-len(
user_base_domain
):] != user_base_domain and user_base_domain != '.' + zone[
'domain']:
is_valid = False
errors.append({
'row':
zone['row'],
'error':
'Zone {0} does not match your assigned master domain'.
format(zone['domain'])
})
if is_valid:
domain = {
'id':
domain_mapping[zone['domain']]
if zone['domain'] in domain_mapping else 0,
'domain':
zone['domain'],
'active':
active,
'catch_all':
catch_all,
'forwarding':
forwarding,
'regex':
regex,
'master':
master,
'tags':
tags
}
items.append(domain)
return items, errors
def main():
seeding()
# number of parallel agents
parallel_envs = 4
# number of training episodes.
# change this to higher number to experiment. say 30000.
number_of_episodes = 1000
episode_length = 80
batchsize = 1000
# how many episodes to save policy and gif
save_interval = 1000
t = 0
# amplitude of OU noise
# this slowly decreases to 0
noise = 2
noise_reduction = 0.9999
# how many episodes before update
episode_per_update = 2 * parallel_envs
log_path = os.getcwd() + "/log"
model_dir = os.getcwd() + "/model_dir"
os.makedirs(model_dir, exist_ok=True)
torch.set_num_threads(parallel_envs)
env = envs.make_parallel_env(parallel_envs)
# keep 5000 episodes worth of replay
buffer = ReplayBuffer(int(5000 * episode_length))
# initialize policy and critic
maddpg = MADDPG()
logger = SummaryWriter(log_dir=log_path)
agent0_reward = []
agent1_reward = []
agent2_reward = []
# training loop
# show progressbar
widget = [
'episode: ',
pb.Counter(), '/',
str(number_of_episodes), ' ',
pb.Percentage(), ' ',
pb.ETA(), ' ',
pb.Bar(marker=pb.RotatingMarker()), ' '
]
timer = pb.ProgressBar(widgets=widget, maxval=number_of_episodes).start()
# use keep_awake to keep workspace from disconnecting
for episode in range(0, number_of_episodes + parallel_envs, parallel_envs):
timer.update(episode)
reward_this_episode = np.zeros((parallel_envs, 3))
all_obs = env.reset()
obs, obs_full = transpose_list(all_obs)
# for calculating rewards for this particular episode - addition of all time steps
# save info or not
save_info = (episode % save_interval < parallel_envs)
frames = []
tmax = 0
if save_info:
frames.append(env.render('rgb_array'))
for episode_t in range(episode_length):
t += parallel_envs
# explore = only explore for a certain number of episodes
# action input needs to be transposed
actions = maddpg.act(transpose_to_tensor(obs), noise=noise)
noise *= noise_reduction
actions_array = torch.stack(actions).detach().numpy()
# transpose the list of list
# flip the first two indices
# input to step requires the first index to correspond to number of parallel agents
actions_for_env = np.rollaxis(actions_array, 1)
# step forward one frame
next_obs, next_obs_full, rewards, dones, info = env.step(
actions_for_env)
# add data to buffer
transition = (obs, obs_full, actions_for_env, rewards, next_obs,
next_obs_full, dones)
buffer.push(transition)
reward_this_episode += rewards
obs, obs_full = next_obs, next_obs_full
# save gif frame
if save_info:
frames.append(env.render('rgb_array'))
tmax += 1
# update once after every episode_per_update
if len(buffer
) > batchsize and episode % episode_per_update < parallel_envs:
for a_i in range(3):
samples = buffer.sample(batchsize)
maddpg.update(samples, a_i, logger)
maddpg.update_targets(
) # soft update the target network towards the actual networks
for i in range(parallel_envs):
agent0_reward.append(reward_this_episode[i, 0])
agent1_reward.append(reward_this_episode[i, 1])
agent2_reward.append(reward_this_episode[i, 2])
if episode % 100 == 0 or episode == number_of_episodes - 1:
avg_rewards = [
np.mean(agent0_reward),
np.mean(agent1_reward),
np.mean(agent2_reward)
]
agent0_reward = []
agent1_reward = []
agent2_reward = []
for a_i, avg_rew in enumerate(avg_rewards):
logger.add_scalar('agent%i/mean_episode_rewards' % a_i,
avg_rew, episode)
# saving model
save_dict_list = []
if save_info:
for i in range(3):
save_dict = {
'actor_params':
maddpg.maddpg_agent[i].actor.state_dict(),
'actor_optim_params':
maddpg.maddpg_agent[i].actor_optimizer.state_dict(),
'critic_params':
maddpg.maddpg_agent[i].critic.state_dict(),
'critic_optim_params':
maddpg.maddpg_agent[i].critic_optimizer.state_dict()
}
save_dict_list.append(save_dict)
torch.save(
save_dict_list,
os.path.join(model_dir, 'episode-{}.pt'.format(episode)))
# save gif files
imageio.mimsave(os.path.join(model_dir,
'episode-{}.gif'.format(episode)),
frames,
duration=.04)
env.close()
logger.close()
timer.finish()
attack=args.A,
cntmodes=c.eigenvectors,
t0=args.pt0)
# 0.6) Define saving paths:
sr = SaveResults(data=d, cnt=c, pert=p, system=d.system, parameters=opts)
# 0.7) Other theoretical tools:
th = TheoreticalComputations(d, c, p)
F = FrequencySpectrum()
# Progress-bar configuration
widgets = [
'Progress: ',
pb.Percentage(), ' ',
pb.Bar(marker=pb.RotatingMarker()), ' ',
pb.ETA(), ' '
]
###################################################################################
# 1) Simulation (Integrate the system)
print('Simulating ...')
pbar = pb.ProgressBar(widgets=widgets, maxval=10 * (d.nsteps + 1)).start()
time1 = timer()
tstep = 0
temps = 0
nois = 0.0
noiseinput = 0.0
kp = k = 0
tgamma = 0
def __import_zones(self,
zones,
user_id,
show_progressbar=False,
batch_size=100):
"""
This function has been heavily optimised as when I tried to import 250k domains its ETA was 1.5h, which isn't
very practical. The main assumption made here is that when this function is called, all validation checks will
have ready been completed.
"""
widget = [
progressbar.FormatLabel(''), ' ',
progressbar.Percentage(), ' ',
progressbar.Bar('#'), ' ',
progressbar.RotatingMarker(), ' ',
progressbar.ETA()
]
count = 0
unique_tags = []
if show_progressbar:
widget[0] = progressbar.FormatLabel('Importing zones')
bar = progressbar.ProgressBar(max_value=len(zones), widgets=widget)
# with bar as zones:
for zone_to_import in list(zones):
count += 1
bar.update(count) if show_progressbar else False
self.__zone_update_or_create(zone_to_import['domain'],
zone_to_import['active'],
zone_to_import['catch_all'],
zone_to_import['forwarding'],
zone_to_import['regex'],
zone_to_import['master'],
user_id,
id=zone_to_import['id'],
autocommit=False)
if count % batch_size == 0:
db.session.commit()
unique_tags = list(set(unique_tags + zone_to_import['tags']))
db.session.commit()
if show_progressbar:
widget[0] = progressbar.FormatLabel('Re-mapping zones')
bar = progressbar.ProgressBar(max_value=len(zones), widgets=widget)
domain_mapping = self.__get_domain_mapping(user_id)
zone_ids = []
i = 0
for zone_to_import in list(zones):
i += 1
bar.update(i) if show_progressbar else False
zone_to_import['id'] = domain_mapping[zone_to_import[
'domain']] if zone_to_import['domain'] in domain_mapping else 0
zone_ids.append(zone_to_import['id'])
self.__zone_clear_tags(zone_ids,
show_progressbar=show_progressbar,
widget=widget)
if show_progressbar:
widget[0] = progressbar.FormatLabel('Importing tags')
bar = progressbar.ProgressBar(max_value=len(zones), widgets=widget)
self.__tags_create(user_id, unique_tags)
tag_mapping = self.__get_tag_mapping(user_id)
count = 0
for zone_to_import in list(zones):
count += 1
bar.update(count) if show_progressbar else False
tags = {}
for tag in zone_to_import['tags']:
tags[tag] = tag_mapping[tag]
self.__zone_save_tags(zone_to_import['id'], tags, autocommit=False)
if count % batch_size == 0:
db.session.commit()
db.session.commit()
return True
def find_all_files_in_directory(AFileClass,
root_dir,
excluded_directories,
search_extensions,
gauge_update_function=None):
"""Recursively searches a directory for files. search_extensions is a dictionary of extension lists"""
global TEXT_FILE_SIZE_LIMIT
all_extensions = [
ext for ext_list in search_extensions.values() for ext in ext_list
]
extension_types = {}
for ext_type, ext_list in search_extensions.iteritems():
for ext in ext_list:
extension_types[ext] = ext_type
if not gauge_update_function:
pbar_widgets = [
'Doc Hunt: ',
progressbar.Percentage(), ' ',
progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
progressbar.ETA(),
progressbar.FormatLabel(' Docs:0')
]
pbar = progressbar.ProgressBar(widgets=pbar_widgets).start()
else:
gauge_update_function(caption='Doc Hunt: ')
doc_files = []
root_dir_dirs = None
root_items_completed = 0
docs_found = 0
for root, sub_dirs, files in os.walk(root_dir):
sub_dirs[:] = [
check_dir for check_dir in sub_dirs if os.path.join(
root, check_dir).lower() not in excluded_directories
]
if not root_dir_dirs:
root_dir_dirs = [
os.path.join(root, sub_dir) for sub_dir in sub_dirs
]
root_total_items = len(root_dir_dirs) + len(files)
if root in root_dir_dirs:
root_items_completed += 1
if not gauge_update_function:
pbar_widgets[6] = progressbar.FormatLabel(' Docs:%s' %
docs_found)
pbar.update(root_items_completed * 100.0 / root_total_items)
else:
gauge_update_function(value=root_items_completed * 100.0 /
root_total_items)
for filename in files:
if root == root_dir:
root_items_completed += 1
afile = AFileClass(filename, root) # AFile or PANFile
if afile.ext.lower() in all_extensions:
afile.set_file_stats()
afile.type = extension_types[afile.ext.lower()]
if afile.type in ('TEXT', 'SPECIAL'
) and afile.size > TEXT_FILE_SIZE_LIMIT:
afile.type = 'OTHER'
afile.set_error(
'File size {1} over limit of {0} for checking'.format(
get_friendly_size(TEXT_FILE_SIZE_LIMIT),
afile.size_friendly()))
doc_files.append(afile)
if not afile.errors:
docs_found += 1
if not gauge_update_function:
pbar_widgets[6] = progressbar.FormatLabel(' Docs:%s' %
docs_found)
pbar.update(root_items_completed * 100.0 /
root_total_items)
else:
gauge_update_function(value=root_items_completed * 100.0 /
root_total_items)
if not gauge_update_function:
pbar.finish()
return doc_files
def main():
seeding()
# number of parallel agents
parallel_envs = 4
# number of training episodes.
# change this to higher number to experiment. say 30000.
number_of_episodes = 1000
episode_length = 80
batchsize = 1000
# how many episodes to save policy and gif
save_interval = 1000
t = 0
# amplitude of OU noise
# this slowly decreases to 0
noise = 2
noise_reduction = 0.9999
# how many episodes before update
episode_per_update = 2 * parallel_envs
log_path = os.getcwd() + '/log'
model_dir = os.getcwd() + '/model_dir'
os.makedirs(model_dir, exist_ok=True)
torch.set_num_threads(parallel_envs)
env = envs.make_parallel_env(parallel_envs)
# keep 5000 episodes worth of replay
buffer = ReplayBuffer(int(5000 * episode_length))
# initialize policy and critic
maddpg = MADDPG()
logger = SummaryWriter(log_dir = log_path)
agent0_reward = []
agent1_reward = []
agent2_reward = []
# training loop
# show progressbar
import progressbar as pb
widget = ['episode: ', pb.Counter(), '/', str(number_of_episodes), ' ',
pb.Percentage(), ' ', pb.ETA(), ' ', pb.Bar(marker=pb.RotatingMarker()), ' ' ]
timer = pb.ProgressBar(widgets=widget, maxval=number_of_episodes).start()
# use keep_awake to keep workspace from disconnecting
# for episode in keep_awake(range(0, number_of_episodes, parallel_envs)):
for episode in range(0, number_of_episodes, parallel_envs):
timer.update(episode)
reward_this_episode = np.zeros((parallel_envs, 3))
all_obs = env.reset()
obs, obs_full = transpose_list(all_obs)
# for calculating rewards for this particular episode - addition of all time steps
# save info or not
save_info = ((episode) % save_interval < parallel_envs or episode==number_of_episodes-parallel_envs)
frames = []
tmax = 0
if save_info:
frames.append(env.render('rgb_array'))
for i in range(0, len(img_dict)):
try:
if img_dict.items()[i].index(ref_img) == 0:
ref_img_index = i
break
except:
pass
LOG.info("Reference image is '" + ref_img + "' at index '" + str(ref_img_index) + "'")
max_left_offset = 0
max_right_offset = 0
max_top_offset = 0
max_bottom_offset = 0
pbar_widgets = [progressbar.FormatLabel(''), ' ', progressbar.Percentage(), ' ', progressbar.Bar('#'), ' ', progressbar.RotatingMarker()]
pbar = progressbar.ProgressBar(widgets = pbar_widgets, maxval = len(img_dict) - 1).start()
pbar_counter = 0
# Start from the reference picture and process backwards
resize = 0.25
max_shake = 1 / resize / wide_side * 100
# TODO: Move the contents of the for loops in a new function
# TODO: Save the pictures if negative xShift and yShift values are chosen
for i in range(ref_img_index, 0, -1):
i = i - 1
pbar_counter = pbar_counter + 1
pbar_widgets[0] = progressbar.FormatLabel("Processing image '{0}' ({1}/{2})".format(img_dict.items()[i][0], pbar_counter, len(img_dict) - 1))
pbar.update(pbar_counter)
temp_xShift, temp_yShift = find_image_shift(img_dict.items()[i+1][1]['fullpath'], img_dict.items()[i][1]['fullpath'], max_shake = options.max_shake, resize=resize)
# The first run is resizing the pictures to 25% for a rough quick parse so we get a temp_xShift and temp_yShift.
# After we find the temp shifts, we know that the real shifts are (1 / resize / wide_side) * 100 percent pixels around the temp shifts * 1 / resize, so rerun the find image shift with resize = 1 this time.
def main(argv):
"""Check flags and start the threaded push."""
files = FLAGS(argv)[1:]
# Flags "devices" and "devices_from_filenames" are mutually exclusive.
if ((not FLAGS.targets and not FLAGS.devices_from_filenames)
or (FLAGS.targets and FLAGS.devices_from_filenames)):
raise UsageError('No targets defined, try --targets.')
# User must provide a vendor.
elif not FLAGS.vendor:
raise UsageError(
'No vendor defined, try the --vendor flag (i.e. --vendor ios)')
# We need some configuration files unless --command is used.
elif not files and not FLAGS.command:
raise UsageError(
'No configuration files provided. Provide these via argv / glob.')
else:
# Vendor implementations must be named correctly, i.e. IosDevice.
vendor_classname = FLAGS.vendor.capitalize() + 'Device'
class_path = '.'.join([FLAGS.vendor.lower(), vendor_classname])
try:
pusher = eval(class_path)
except NameError:
raise UsageError(
'The vendor "%s" is not implemented or imported. Please select a '
'valid vendor' % FLAGS.vendor)
if not FLAGS.user:
FLAGS.user = getpass.getuser()
if FLAGS.devices_from_filenames:
FLAGS.targets = [os.path.basename(x) for x in files]
print 'Ready to push per-device configurations to %s' % FLAGS.targets
else:
print 'Ready to push %s to %s' % (files
or FLAGS.command, FLAGS.targets)
passw = getpass.getpass('Password:'******'Pushing... ',
progressbar.Percentage(), ' ',
progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
progressbar.ETA(), ' ',
progressbar.FileTransferSpeed()
]
pbar = progressbar.ProgressBar(widgets=widgets).start()
for counter, device in enumerate(FLAGS.targets):
if FLAGS.command:
thread = PushThread(device, FLAGS.command, pusher, passw)
else:
consolidated = JoinFiles(files)
thread = PushThread(device, consolidated, pusher, passw)
thread.start()
pbar.update((len(FLAGS.targets) / 100.0) * counter)
pbar.finish()
def launch_group_by_genome(all_genomes, all_alns, status, outfile, dname,
type_ali, quiet):
"""
Function calling group_by_genome in a pool, while giving information to user
(time elapsed)
Parameters
----------
all_genomes : []
list of all genomes in the dataset
all_alns : str
path to file containing all alignments concatenated
status : str
"OK" if concatenation file already existed before running, "Done" if just did concatenation
outfile : str
file containing all families align by genome
dname : str
name of dataset
type_ali : str
nucleic or protein
quiet : bool
True if nothing must be sent to sdtout/stderr, False otherwise
Returns
-------
bool
- True if everything went well or was already done
- False if error occurred in at least one step
"""
# Status = Done means that we just did the concatenation. So, if grouped by genome
# file already exists, remove it.
if status == "Done":
if os.path.isfile(outfile):
utils.remove(outfile)
# Status was not 'Done' (it was 'ok', concat file already existed). And by_genome file
# also already exists. Warn user
if os.path.isfile(outfile):
logger.info(f"{type_ali} alignments already grouped by genome")
logger.warning(
(f"{type_ali} alignments already grouped by genome in {outfile}. "
"Program will end. "))
return True
logger.info(f"Grouping {type_ali} alignments per genome")
bar = None
if not quiet:
widgets = [
progressbar.BouncingBar(marker=progressbar.RotatingMarker(
markers="◐◓◑◒")), " - ",
progressbar.Timer()
]
bar = progressbar.ProgressBar(widgets=widgets,
max_value=20,
term_width=50)
pool = multiprocessing.Pool(1)
args = [all_genomes, all_alns, outfile]
final = pool.map_async(group_by_genome, [args], chunksize=1)
pool.close()
if not quiet:
while True:
if final.ready():
break
bar.update()
bar.finish()
pool.join()
return False not in final.get()
def manual_run_functionality_all(threads=8, optimize=False):
# Grab all binaries under binaries-private/cgc_samples_multiflags, and reassemble them
binaries = []
for dirname, dirlist, filelist in os.walk(
os.path.join(bin_location, 'cgc_samples_multiflags')):
for b in filelist:
if '.' in b:
continue
p = os.path.normpath(
os.path.join('cgc_samples_multiflags', dirname, b))
binaries.append(p)
random.shuffle(binaries)
if threads > 1:
manager = Manager()
queue = manager.Queue()
results = []
pool = Pool(threads, maxtasksperchild=40)
progress = progressbar.ProgressBar(widgets=[
progressbar.Bar(marker=progressbar.RotatingMarker()),
' ',
progressbar.Percentage(),
' ',
progressbar.Timer(),
' ',
progressbar.ETA(),
],
maxval=len(binaries))
progress.start()
pool.map_async(partial(manual_run_functionality_core,
optimize=optimize,
queue=queue),
binaries,
chunksize=1)
pool.close()
while len(results) != len(binaries):
time.sleep(0.5)
progress.update(len(results))
# read result from queue
try:
data = queue.get_nowait()
except Queue.Empty:
continue
results.append(data)
progress.finish()
# statistics
for b, r, exc in results:
if not r:
print "Fail to process %s: %s" % (b, str(exc))
else:
for b in binaries:
manual_run_functionality_core(b, optimize=optimize)
import pymongo
from pymongo import MongoClient
import sys
import progressbar as pb
widgets = ['Time:', pb.Percentage(), ' ',pb.Bar(marker=pb.RotatingMarker()), ' ', pb.ETA()]
#timer = pb.ProgressBar(widgets=widgets, maxval=8100000).start()
client= MongoClient('mongodb://*****:*****@192.168.2.75/bgpPaths', 27017)
db=client.bgpPaths
collection=db.bgpNeighs
if(len(sys.argv)<2):
print("Error-Enter Command in format: python relate.py <relationship-file-name>")
sys.exit(0)
with open(sys.argv[1]) as bgp_relations:
# time=0
for relation in bgp_relations:
relation_array=relation.split('|')
as_data=collection.find_one({'as':relation_array[0]})
neighbour_data=collection.find_one({'as':relation_array[1]})
if as_data==None:
new_as_data={
'as':relation_array[0],
'neighbours':{
'siblings':[],
'customers':[],
# >>> g.add_proxy("knows", Knows)
# >>> james = g.people.create(name="James")
# >>> julie = g.people.create(name="Julie")
# >>> g.knows.create(james, julie)
import progressbar as pb
N = 150000
for N, ln in enumerate(wn.all_lemma_names()):
pass
widgets = [
pb.Counter(),
'%d rows: ' % N,
pb.Percentage(), ' ',
pb.RotatingMarker(), ' ',
pb.Bar(), ' ',
pb.ETA()
]
pbar = pb.ProgressBar(widgets=widgets, maxval=N).start()
for i, ln in enumerate(wn.all_lemma_names()):
pbar.update(i)
lemma = g.lemma.create(name=str(ln))
#Lemma(ln).save()
pb.finish()
# class Knows(Relationship):
# label = "knows"
def train(maddpg, env, n_episodes=1000, save_every=50):
"""Training loop helper for running the environment using the MADDPG algorithm.
Params
======
maddpg (MADDPG): instance of MADDPG wrapper class
env (UnityEnvironment): instance of Unity environment for training
n_episodes (int): number of episodes to train for
save_every (int): frequency to save model weights
"""
widget = [
"Episode: ", pb.Counter(), '/' , str(n_episodes), ' ',
pb.Percentage(), ' ', pb.ETA(), ' ', pb.Bar(marker=pb.RotatingMarker()), ' ',
'Rolling Average: ', pb.FormatLabel('')
]
timer = pb.ProgressBar(widgets=widget, maxval=n_episodes).start()
solved = False
scores_total = []
scores_deque = deque(maxlen=100)
rolling_score_averages = []
last_best_score = 0.0
# Environment information
brain_name = env.brain_names[0]
for i_episode in range(1, n_episodes+1):
current_average = 0.0 if i_episode == 1 else rolling_score_averages[-1]
widget[12] = pb.FormatLabel(str(current_average)[:6])
timer.update(i_episode)
env_info = env.reset(train_mode=True)[brain_name]
states = env_info.vector_observations[:, -STATE_SIZE:]
scores = np.zeros(NUM_AGENTS)
maddpg.reset()
while True:
actions = maddpg.act(states)
env_info = env.step(actions)[brain_name]
next_states = env_info.vector_observations[:, -STATE_SIZE:]
rewards = env_info.rewards
dones = env_info.local_done
maddpg.step(states, actions, rewards, next_states, dones)
scores += rewards
states = next_states
if np.any(dones):
break
max_episode_score = np.max(scores)
scores_deque.append(max_episode_score)
scores_total.append(max_episode_score)
average_score = np.mean(scores_deque)
rolling_score_averages.append(average_score)
if i_episode % save_every == 0:
maddpg.save_model()
if average_score >= 0.5 and not solved:
print('\nEnvironment solved in {:d} episodes!\tAverage Score: {:.2f}'.format(
i_episode, average_score
))
solved = True
maddpg.save_model()
last_best_score = average_score
break
if average_score >= 1 % save_every == 0:
print('\nStopped at {:d} episodes!\tAverage Score: {:.2f}'.format(i_episode, average_score))
maddpg.save_model()
break
if i_episode % save_every == 0 and solved:
# Only save these weights if they are better than the ones previously saved
if average_score > last_best_score:
last_best_score = average_score
maddpg.save_model()
return scores_total, rolling_score_averages
def main(args, unknown_args):
'''
main func: pass cmd input and simulate pedigree samples
'''
def cleanup(items):
for item in ['{}.{}'.format(args.outfile, x) for x in items]:
os.system('/bin/rm -rf {}'.format(item))
## check user input
checkInput(args)
## print header
if args.print_header:
print(
"gene1,gene2,plod1,nlod1,phlod1,nhlod1,plod2,nlod2,phlod2,nhlod2,prop1,prop2,moi,ahet,fam_size,min_offspring,max_offspring"
)
sys.exit()
## set seed
if not args.seed:
args.seed = time.time()
random.seed(args.seed)
## update proportions of number of offspring
offNumProp = updateOffNumProp(args.offspring)
## parse input gene info
gene1, gene2 = parseGeneInfo(args.genes[0],
recRate=args.recrate), parseGeneInfo(
args.genes[1], recRate=args.recrate)
## clean up directory
cleanup(['lods', 'hlods'])
## simulation
counter = {}
alphas = {'lods': args.alpha[0], 'hlods': args.alpha[1]}
if not args.debug:
pbar = progressbar.ProgressBar(widgets=[
'Simulation [{}]'.format(args.seed), ' ',
progressbar.Percentage(), ' ',
progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
progressbar.ETA(), ' '
],
maxval=int(args.numreps)).start()
for i in xrange(1, args.numreps + 1):
samples = []
if args.allelichet:
diseaseVariantIndices = None
else:
diseaseVariantIndices = [
gene1['d_idx'][weightedRandomIdx(gene1['cumuProbs_dMaf'])],
gene2['d_idx'][weightedRandomIdx(gene2['cumuProbs_dMaf'])]
]
for j in xrange(args.numfamilies):
numOffspring = getNumOffspring(offNumProp)
pedInfo = simPedigree([gene1, gene2],
numOffspring,
args.mode,
args.locusheterogenprop,
diseaseVariantIndices,
familyID=j + 1,
recRate=args.recrate,
ifMarkerVar=args.if_marker_var,
fakeLD=args.ld,
parentMissing=args.par_missing)
samples.extend(pedInfo)
# write *.fam file per sample for pedigree structure info only
# write *.vcf file per sample for variant info
dirName, baseName = os.path.dirname(args.outfile), os.path.basename(
args.outfile)
fam = os.path.join(dirName, baseName + 'rep' + str(i) + ".fam")
vcf = os.path.join(dirName, baseName + 'rep' + str(i) + ".vcf")
writeVCF(samples, writePedsToFile(samples, fam, pedStructOnly=True),
gene1, gene2, vcf)
# also write *.ped file if args.save is true
if args.save:
ped = os.path.join(dirName, baseName + 'rep' + str(i) + ".ped")
writePedsToFile(samples, ped, pedStructOnly=False)
if args.sim_only:
if not args.debug:
pbar.update(i)
continue
#if not args.save:
# cleanup(['vcf.gz', 'vcf.gz.tbi'])
vcf = indexVCF(vcf, verbose=False)
# linkage analysis
cmd = "seqlink --vcf {} --fam {} --output {} {} 2> /dev/null".\
format(vcf, fam, args.outfile, " ".join(unknown_args))
os.system(cmd)
res = {'lods': {}, 'hlods': {}}
for score in ['lods', 'hlods']:
for fn in glob.glob('{}/heatmap/*.{}'.format(args.outfile, score)):
for marker, theta, value in zip(getColumn(fn, 1),
getColumn(fn, -2),
getColumn(fn, -1)):
value = float(value)
# convert single SNV marker to gene marker
if ":" in marker:
marker = GENEMAP[marker.split(":")[0]]
if marker not in res[score]:
res[score][marker] = value
else:
if theta == '0.0':
res[score][marker] = value
# write result to file and calculate significance
for score in res:
with open(args.outfile + '.{}'.format(score), 'a') as f:
for marker in res[score]:
f.write('{}\t{}\n'.format(marker, res[score][marker]))
if marker not in counter:
counter[marker] = {'lods': [0, 0], 'hlods': [0, 0]}
if res[score][marker] >= alphas[score]:
counter[marker][score][0] += 1
counter[marker][score][1] += 1
else:
counter[marker][score][1] += 1
if not args.save:
os.system('rm -rf {0}/ {0}rep{1}*'.format(args.outfile, i))
if not args.debug:
pbar.update(i)
if not args.debug:
pbar.finish()
# report power calculation
gs = [x.split('.')[0] for x in args.genes]
out = [
gs[0], gs[1], counter[gs[0]]['lods'][0] /
float(counter[gs[0]]['lods'][1] + 1E-10) if gs[0] in counter else -9,
counter[gs[0]]['lods'][1] if gs[0] in counter else -9,
counter[gs[0]]['hlods'][0] /
float(counter[gs[0]]['hlods'][1] + 1E-10) if gs[0] in counter else -9,
counter[gs[0]]['hlods'][1] if gs[0] in counter else -9,
counter[gs[1]]['lods'][0] /
float(counter[gs[1]]['lods'][1] + 1E-10) if gs[1] in counter else -9,
counter[gs[1]]['lods'][1] if gs[1] in counter else -9,
counter[gs[1]]['hlods'][0] /
float(counter[gs[1]]['hlods'][1] + 1E-10) if gs[1] in counter else -9,
counter[gs[1]]['hlods'][1] if gs[1] in counter else -9,
args.locusheterogenprop[0], args.locusheterogenprop[1], args.mode,
int(args.allelichet), args.numfamilies, args.offspring[0],
args.offspring[1]
]
print ','.join(map(str, out))
return 0
def main(args):
run_date = time.time()
# Configure Logging
root = logging.getLogger()
if(args.debug):
root.setLevel(logging.DEBUG)
else:
root.setLevel(logging.INFO)
if args.quiet:
root.propogate = False
# Set up the database.
pgdb = DBUtils(config_file=DB_CONFIG_FILE)
# Get the name of this agent trail for later use
at = AgentTrail()
at.readTrail(args.trail, DB_CONFIG_FILE)
trail_name = at.getName()
if not args.quiet and not args.debug and not args.script_mode:
try:
TOTAL_GENERATIONS = (len(args.network) *
args.generations * args.repeat)
widgets = ['Processed: ', progressbar.Percentage(), ' ',
progressbar.Bar(marker=progressbar.RotatingMarker()),
' ', progressbar.ETA()]
pbar = progressbar.ProgressBar(
widgets=widgets,
maxval=TOTAL_GENERATIONS).start()
except:
pbar = None
else:
pbar = None
current_overall_gen = 0
for curr_network in args.network:
# Query the database to get the network information.
pybrain_network = pgdb.getNetworkByID(curr_network)
temp_f_h, temp_f_network = tempfile.mkstemp()
os.close(temp_f_h)
with open(temp_f_network, "w") as f:
pickle.dump(pybrain_network, f)
# TODO: Need to fix this for chemistry support here.
if "Chemical" in pybrain_network.name:
chem_re = re.compile(
"JL NN Chemical DL([0-9]+) \([0-9]+,[0-9]+,[0-9]+\) v[0-9]+")
chem_dl_length = int(chem_re.findall(pybrain_network.name)[0])
network_params_len = len(pybrain_network.params) + chem_dl_length * 3
else:
network_params_len = len(pybrain_network.params)
# Query the database to get the trail information.
(data_matrix,
db_trail_name,
init_rot) = pgdb.getTrailData(args.trail)
# Calculate the maximum amount of food for potential later comparison.
MAX_FOOD = np.bincount(np.array(data_matrix).flatten())[1]
for curr_repeat in range(0, args.repeat):
repeat_start_time = datetime.datetime.now()
gens_stat_list = [None] * args.generations
# Create an empty array to store the launches for SCOOP.
launches = []
# Prepare the array for storing hall of fame.
hof_array = np.zeros((args.generations,
network_params_len))
toolbox = base.Toolbox()
toolbox.register("map", scoop.futures.map)
toolbox.register("attr_float", random.uniform,
a=args.weight_min, b=args.weight_max)
toolbox.register("individual", tools.initRepeat, creator.Individual,
toolbox.attr_float, n=network_params_len)
toolbox.register("population", tools.initRepeat, list,
toolbox.individual)
an_temp = AgentNetwork()
an_temp.readNetworkFromFile(temp_f_network)
at_temp = AgentTrail()
at_temp.readTrailInstant(data_matrix, db_trail_name, init_rot)
toolbox.register("evaluate", __singleMazeTask, moves=args.moves,
network=pickle.dumps(an_temp), trail=pickle.dumps(at_temp))
toolbox.register("mate", tools.cxTwoPoint)
if args.mutate_type == 1:
toolbox.register("mutate",
tools.mutFlipBit,
indpb=P_BIT_MUTATE)
elif args.mutate_type == 2:
toolbox.register("mutate",
mutUniformFloat,
low=args.weight_min,
up=args.weight_max,
indpb=P_BIT_MUTATE)
elif args.mutate_type == 3:
toolbox.register("mutate",
mutUniformFloat,
low=args.weight_min,
up=args.weight_max,
indpb=0.30)
elif args.mutate_type == 4:
toolbox.register("mutate",
mutUniformFloat,
low=args.weight_min,
up=args.weight_max,
indpb=0.10)
elif args.mutate_type == 5:
toolbox.register("mutate",
tools.mutGaussian,
mu=0,
indpb=0.05)
else:
print "ERROR: Please selct a valid mutate type!"
sys.exit(10)
if args.selection == 1:
# Selection is tournment. Must use argument from user.
toolbox.register("select", tools.selTournament,
tournsize=args.tournament_size)
elif args.selection == 2:
toolbox.register("select", tools.selRoulette)
elif args.selection == 3:
toolbox.register("select", tools.selNSGA2)
elif args.selection == 4:
toolbox.register("select", tools.selSPEA2)
elif args.selection == 5:
toolbox.register("select", tools.selRandom)
elif args.selection == 6:
toolbox.register("select", tools.selBest)
elif args.selection == 7:
toolbox.register("select", tools.selWorst)
elif args.selection == 8:
toolbox.register("select", tools.selTournamentDCD)
else:
print "ERROR: Something is wrong with selection method!"
sys.exit(10)
# Start a new evolution
population = toolbox.population(n=args.population)
halloffame = tools.HallOfFame(maxsize=1)
food_stats = tools.Statistics(key=lambda ind: ind.fitness.values[0])
move_stats = tools.Statistics(key=lambda ind: ind.fitness.values[1])
mstats = tools.MultiStatistics(food=food_stats, moves=move_stats)
mstats.register("min", np.min)
mstats.register("avg", np.mean)
mstats.register("max", np.max)
mstats.register("std", np.std)
mstats.register("mode", mode)
# Record the start of this run.
log_time = datetime.datetime.now()
# Evaluate and record the first generation here.
invalid_ind = [ind for ind in population if not ind.fitness.valid]
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
# Determine the current generations statistics.
record = mstats.compile(population)
if args.debug:
print "DEBUG: Completed generation 1"
hof_indiv = np.array(tools.selBest(population, k=1)[0])
hof_array[0] = hof_indiv
# Add the hall of fame to launches.
launches.append(
scoop.futures.submit(__singleMazeTask,
hof_indiv,
args.moves,
pickle.dumps(an_temp),
pickle.dumps(at_temp),
1,
record)
)
# Keep track of the average food history.
mean_food_history = []
smart_term_msg = ""
# Begin the generational process
for gen in range(2, args.generations + 1):
# Vary the pool of individuals
if args.variation in [1]:
offspring = algorithms.varAnd(population, toolbox,
cxpb=args.prob_crossover, mutpb=args.prob_mutate)
elif args.variation in [2, 3, 4]:
offspring = algorithms.varOr(population, toolbox,
lambda_=args.lambda_,
cxpb=args.prob_crossover, mutpb=args.prob_mutate)
elif args.variation in [5]:
# Take and modify the varAnd from DEAP.
offspring = [toolbox.clone(ind) for ind in population]
# Apply crossover and mutation on the offspring
for i in range(1, len(offspring), 2):
if random.random() < args.prob_crossover:
offspring[i-1], offspring[i] = toolbox.mate(
offspring[i-1], offspring[i])
del (offspring[i-1].fitness.values,
offspring[i].fitness.values)
for i in range(len(offspring)):
if random.random() < args.prob_mutate:
if args.mutate_type in [5]:
offspring[i], = toolbox.mutate(
offspring[i],
sigma=np.std(offspring[i]))
else:
offspring[i], = toolbox.mutate(
offspring[i], offspring[i])
del offspring[i].fitness.values
else:
print ("ERROR: Something is really wrong! " +
"Reached an invalid variation type!")
sys.exit(5)
# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
# Update the hall of fame with the generated individuals
if halloffame is not None:
halloffame.update(offspring)
# Replace the current population by the offspring
if args.variation in [2, 3]:
population[:] = toolbox.select(offspring, args.population)
elif args.variation in [4, 5]:
population[:] = toolbox.select(offspring + population,
args.population)
else:
population[:] = offspring
# Determine the current generations statistics.
record = mstats.compile(population)
if args.debug:
print "DEBUG: Completed generation {0}.".format(gen)
print (
"DEBUG: Food (Min / Max / Avg / Std / Mode): "
"{0} / {1} / {2} / {3} / {4}".format(
record["food"]["min"],
record["food"]["max"],
record["food"]["avg"],
record["food"]["std"],
record["food"]["mode"]))
print (
"DEBUG: Moves (Min / Max / Avg / Std / Mode): "
"{0} / {1} / {2} / {3} / {4}".format(
record["moves"]["min"],
record["moves"]["max"],
record["moves"]["avg"],
record["moves"]["std"],
record["moves"]["mode"]))
hof_indiv = np.array(tools.selBest(population, k=1)[0])
hof_array[gen - 1] = hof_indiv
# Add the hall of fame to launches.
launches.append(
scoop.futures.submit(__singleMazeTask,
hof_indiv,
args.moves,
pickle.dumps(an_temp),
pickle.dumps(at_temp),
gen,
record)
)
# Update the mean food history.
mean_food_history.append(record["food"]["avg"])
# Update the progress bar
if pbar:
current_overall_gen += 1
pbar.update(current_overall_gen)
# Check if it is time to quit if variation is 3. Critera are
# any of the following:
# 1) All food has been collected.
# 2) Mean has not changed for args.mean_check_length
# 3) Run out of generations (happens without this if)
if args.variation in [3, 4, 5] and not args.no_early_quit:
if (int(record["food"]["max"]) == int(MAX_FOOD)):
smart_term_msg = ("Exited at generation {0} because "
"all food was consumed.").format(gen)
break
elif(len(mean_food_history) >= args.mean_check_length and
(np.std(mean_food_history[-args.mean_check_length:])
< 0.1)):
smart_term_msg = ("Exited at generation {0} because "
"mean check length has been met.").format(gen)
break
# Evaluate the Hall of Fame individual for each generation here
# in a multithreaded fashion to speed things up.
for this_future in scoop.futures.as_completed(launches):
result = this_future.result()
gens_stat_list[result[0] - 1] = result[1]
# Remove all of the None values from the gen_stat_list
gens_stat_list = filter(lambda a: a is not None, gens_stat_list)
# Record the statistics on this run.
run_info = {}
run_info["trails_id"] = args.trail
run_info["networks_id"] = curr_network
run_info["selection_id"] = args.selection
run_info["mutate_id"] = args.mutate_type
run_info["host_type_id"] = 1 # Only one host type for now.
run_info["variations_id"] = args.variation
run_info["run_date"] = log_time
run_info["hostname"] = socket.getfqdn()
run_info["generations"] = args.generations
run_info["population"] = args.population
run_info["moves_limit"] = args.moves
run_info["sel_tourn_size"] = args.tournament_size
if args.variation in [1, 5]:
run_info["lambda"] = 0
else:
run_info["lambda"] = args.lambda_
run_info["p_mutate"] = args.prob_mutate
run_info["p_crossover"] = args.prob_crossover
run_info["weight_min"] = args.weight_min
run_info["weight_max"] = args.weight_max
run_info["debug"] = args.debug
# Version for if anything changes in python GA Algorithm
run_info["algorithm_ver"] = 2
run_info["mean_check_length"] = args.mean_check_length
run_info["runtime"] = (datetime.datetime.now() -
repeat_start_time)
if not args.disable_db:
run_id = pgdb.recordRun(run_info, gens_stat_list)
else:
run_id = -1
if args.script_mode:
if run_id > 0:
print (
"Completed repeat {0} with run ID {1}. {2}".format(
curr_repeat,
run_id,
smart_term_msg
))
else:
print (
"Completed repeat {0} without logging to DB. {1}".format(
curr_repeat,
smart_term_msg
))
# Delete the temporary file
os.remove(temp_f_network)
# Calculate and display the total runtime
if pbar:
pbar.finish()
total_time_s = time.time() - run_date
if run_id > 0:
print "Final Run ID {0} completed all runs in {1}. {2}".format(
run_id,
time.strftime('%H:%M:%S', time.gmtime(total_time_s)),
smart_term_msg)
else:
print "UNLOGGED Run completed in {0}. {1}".format(
time.strftime('%H:%M:%S', time.gmtime(total_time_s)),
smart_term_msg)
def check_pst_regexs(self,
regexs,
search_extensions,
hunt_type,
gauge_update_function=None):
""" Searches a pst file for regular expressions in messages and attachments using regular expressions"""
all_extensions = search_extensions['TEXT'] + search_extensions[
'ZIP'] + search_extensions['SPECIAL']
if not gauge_update_function:
pbar_widgets = [
'%s Hunt %s: ' % (hunt_type, unicode2ascii(self.filename)),
progressbar.Percentage(), ' ',
progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
progressbar.ETA(),
progressbar.FormatLabel(' %ss:0' % hunt_type)
]
pbar = progressbar.ProgressBar(widgets=pbar_widgets).start()
else:
gauge_update_function(caption='%s Hunt: ' % hunt_type)
try:
apst = pst.PST(self.path)
total_messages = apst.get_total_message_count()
total_attachments = apst.get_total_attachment_count()
total_items = total_messages + total_attachments
items_completed = 0
for folder in apst.folder_generator():
for message in apst.message_generator(folder):
if message.Subject:
message_path = os.path.join(folder.path,
message.Subject)
else:
message_path = os.path.join(folder.path,
u'[NoSubject]')
if message.Body:
self.check_text_regexs(message.Body, regexs,
message_path)
if message.HasAttachments:
for subattachment in message.subattachments:
if get_ext(subattachment.Filename
) in search_extensions[
'TEXT'] + search_extensions['ZIP']:
attachment = message.get_attachment(
subattachment)
self.check_attachment_regexs(
attachment, regexs, search_extensions,
message_path)
items_completed += 1
items_completed += 1
if not gauge_update_function:
pbar_widgets[6] = progressbar.FormatLabel(
' %ss:%s' % (hunt_type, len(self.matches)))
pbar.update(items_completed * 100.0 / total_items)
else:
gauge_update_function(value=items_completed * 100.0 /
total_items)
apst.close()
except IOError:
self.set_error(sys.exc_info()[1])
except pst.PSTException:
self.set_error(sys.exc_info()[1])
if not gauge_update_function:
pbar.finish()
return self.matches
def image_composite(inputs, algo, output, oformat, vza, mask_band, mask_val):
""" Create image composites based on some criteria
Output image composites retain original values from input images that meet
a certain criteria. For example, in a maximum NDVI composite with 10 input
images, all bands for a given pixel will contain the band values from the
input raster that had the highest NDVI value.
Users can choose from a set of predefined compositing algorithms or may
specify an Snuggs S-expression that defines the compositing criteria.
Normalized Differenced indexes can be computed using "(normdiff a b)" for
the Normalized Difference between "a" and "b" (or "nir" and "red").
See https://github.com/mapbox/snuggs for more information on Snuggs
expressions.
The indexes for common optical bands (e.g., red, nir, blue) within the
input rasters are included as optional arguments and are indexed in
wavelength sequential order. You may need to overwrite the default indexes
of bands used in a given S-expression with the correct band index.
Additional bands may be identified and indexed using the
'--band NAME=INDEX' option.
Currently, input images must be "stacked", meaning that they contain the
same bands and are the same shape and extent.
Example:
1. Create a composite based on maximum NDVI
Use the built-in maxNDVI algorithm:
\b
$ image_composite.py --algo maxNDVI image1.gtif image2.gtif image3.gtif
composite_maxNDVI.gtif
or with S-expression:
\b
$ image_composite.py --expr '(max (/ (- nir red) (+ nir red)))'
image1.gtif image2.gtif image3.gtif composite_maxNDVI.gtif
or with S-expressions using the normdiff shortcut:
\b
$ image_composite.py --expr '(max (normdiff nir red))'
image1.gtif image2.gtif image3.gtif composite_maxNDVI.gtif
2. Create a composite based on median EVI (not recommended)
With S-expression:
\b
$ evi='(median (/ (- nir red) (+ (- (+ nir (* 6 red)) (* 7.5 blue)) 1)))'
$ image_composite.py --expr "$evi" image1.gtif image2.gtif image3.gtif
composite_medianEVI.gtif
3. Create a composite based on median NBR
With S-expression:
\b
$ image_composite.py --expr '(median (normdiff nir sswir))'
image1.gtif image2.gtif image3.gtif composite_maxNBR.gtif
"""
verbose = True
if verbose:
logger.setLevel(logging.DEBUG)
elif quiet:
logger.setLevel(logging.ERROR)
expr = _ALGO[algo]
if algo is not None:
logger.debug('Using predefined algorithm: {}'.format(algo))
expr = _ALGO[algo]
# Setup band keywords
_bands = {'vza': vza}
# Find only the band names and indexes required for the composite criteria
crit_indices = {k: v - 1 for k, v in _bands.iteritems() if k in expr}
# Enhance snuggs expressions to return index of value matching function
snuggs.func_map['max'] = lambda a: np.argmax(a, axis=0)
snuggs.func_map['min'] = lambda a: np.argmin(a, axis=0)
snuggs.func_map['median'] = lambda a: np.argmin(
np.abs(a - np.median(a, axis=0)), axis=0)
snuggs.func_map['normdiff'] = lambda a, b: snuggs.eval(
'(/ (- a b) (+ a b))', **{
'a': a,
'b': b
})
with rasterio.drivers():
# Read in the first image to fetch metadata
with rasterio.open(inputs[0]) as first:
meta = first.meta
if 'transform' in meta:
meta.pop('transform') # remove transform since deprecated
meta.update(driver=oformat)
if len(set(first.block_shapes)) != 1:
click.echo('Cannot process input files - '
'All bands must have same block shapes')
raise click.Abort()
block_nrow, block_ncol = first.block_shapes[0]
windows = first.block_windows(1)
n_windows = math.ceil(meta['height'] / block_nrow * meta['width'] /
block_ncol)
# Ensure mask_band exists, if specified
if mask_band:
if mask_band <= meta['count'] and mask_band > 0:
mask_band -= 1
else:
click.echo('Mask band does not exist in INPUT images')
raise click.Abort()
# Initialize output data and create composite
with rasterio.open(output, 'w', **meta) as dst:
# Process by block
dat = np.ma.empty(
(len(inputs), meta['count'], block_nrow, block_ncol),
dtype=np.dtype(meta['dtype']))
mi, mj = np.meshgrid(np.arange(block_nrow),
np.arange(block_ncol),
indexing='ij')
# Open all source files one time
srcs = [rasterio.open(fname) for fname in inputs]
logger.debug('Processing blocks')
if _has_progressbar:
widgets = [
progressbar.Percentage(),
progressbar.BouncingBar(
marker=progressbar.RotatingMarker())
]
pbar = progressbar.ProgressBar(widgets=widgets).start()
for i, (idx, window) in enumerate(windows):
# Update dat and mi, mj only if window changes
nrow = window[0][1] - window[0][0]
ncol = window[1][1] - window[1][0]
if dat.shape[-2] != nrow or dat.shape[-1] != ncol:
dat = np.ma.empty((len(inputs), meta['count'], nrow, ncol),
dtype=np.dtype(meta['dtype']))
mi, mj = np.meshgrid(np.arange(nrow),
np.arange(ncol),
indexing='ij')
for j, src in enumerate(srcs):
dat[j, ...] = src.read(masked=True, window=window)
# Mask values matching mask_vals if mask_band
if mask_band and mask_val:
dat[j, ...].mask = np.logical_or(
dat[j, ...].mask,
np.in1d(
dat[j, mask_band, ...],
mask_val,
).reshape(dat.shape[-2], dat.shape[-1]))
# Find indices of files for composite
crit = {k: dat[:, v, ...] for k, v in crit_indices.iteritems()}
crit_idx = snuggs.eval(expr, **crit)
# Create output composite
# Use np.rollaxis to get (nimage, nrow, ncol, nband) shape
composite = np.rollaxis(dat, 1, 4)[crit_idx, mi, mj]
# Write out
for i_b in range(composite.shape[-1]):
dst.write(composite[:, :, i_b],
indexes=i_b + 1,
window=window)
if _has_progressbar:
pbar.update(int((i + 1) / n_windows * 100))
def main():
seeding()
number_of_episodes = 20000
episode_length = 1000
batchsize = 256
save_interval = 1000
rewards_deque = deque(maxlen=100)
rewards_all = []
noise = 1.0
noise_reduction = 1.0
log_path = os.getcwd() + "/log"
model_dir = os.getcwd() + "/model_dir"
os.makedirs(model_dir, exist_ok=True)
""" Info about the UnityEnvironment
brain_name: 'TennisBrain'
brain: ['brain_name', 'camera_resolutions',
'num_stacked_vector_observations', 'number_visual_observations',
'vector_action_descriptions', 'vector_action_space_size',
'vector_action_space_type', 'vector_observation_space_size',
'vector_observation_space_type']]
"""
env = UnityEnvironment(file_name="Tennis.app")
brain_name = env.brain_names[0]
brain = env.brains[brain_name]
buffer = ReplayBuffer(int(1e5))
# initialize policy and critic
maddpg = MADDPG()
logger = SummaryWriter(log_dir=log_path)
# ------------------------------ training ------------------------------ #
# show progressbar
import progressbar as pb
widget = [
'episode: ',
pb.Counter(), '/',
str(number_of_episodes), ' ',
pb.Percentage(), ' ',
pb.ETA(), ' ',
pb.Bar(marker=pb.RotatingMarker()), ' '
]
timer = pb.ProgressBar(widgets=widget, maxval=number_of_episodes).start()
for episode in range(1, number_of_episodes + 1):
timer.update(episode)
rewards_this_episode = np.zeros((2, ))
""" Info about the UnityEnvironment
env_info: ['agents', 'local_done', 'max_reached', 'memories',
'previous_text_actions', 'previous_vector_actions', 'rewards',
'text_observations', 'vector_observations', 'visual_observations']
actions: List(num_agents=2, action_size=2)
states: List((24,), (24,))
rewards: List(2,)
dones: List(2,)
"""
env_info = env.reset(train_mode=True)[brain_name]
states = env_info.vector_observations
for episode_t in range(episode_length):
# reset the OUNoise for each agent.
for i in range(2):
maddpg.maddpg_agent[i].noise.reset()
actions = maddpg.act(states, noise=noise)
env_info = env.step(actions)[brain_name]
noise *= noise_reduction
next_states = env_info.vector_observations
rewards = env_info.rewards
dones = env_info.local_done
# add data to buffer
transition = (states, actions, rewards, next_states, dones)
buffer.push(transition)
rewards_this_episode += rewards
states = next_states
if any(dones):
break
# update the local and target network
if len(buffer) > batchsize:
# update the local network
for _ in range(5):
for a_i in range(2):
samples = buffer.sample(batchsize)
maddpg.update(samples, a_i, logger)
# soft update the target network
maddpg.update_targets()
rewards_all.append(rewards_this_episode)
rewards_deque.append(np.max(rewards_this_episode))
average_score = np.mean(rewards_deque)
# --------------------- Logging for TensorBoard --------------------- #
logger.add_scalars('rewards', {
'agent0': rewards_this_episode[0],
'agent1': rewards_this_episode[1]
}, episode)
logger.add_scalars('global', {
'score': np.max(rewards_this_episode),
'average_score': average_score
}, episode)
# -------------------------- Save the model -------------------------- #
save_dict_list = []
if episode % save_interval == 0 or average_score >= 0.5:
for i in range(2):
save_dict = \
{'actor_params' : maddpg.maddpg_agent[i].actor.state_dict(),
'actor_optim_params': maddpg.maddpg_agent[i].actor_optimizer.state_dict(),
'critic_params' : maddpg.maddpg_agent[i].critic.state_dict(),
'critic_optim_params' : maddpg.maddpg_agent[i].critic_optimizer.state_dict()}
save_dict_list.append(save_dict)
torch.save(
save_dict_list,
os.path.join(model_dir, 'episode-{}.pt'.format(episode)))
if average_score >= 3.0:
print('\nEnvironment solved in {} episodes!'.format(episode -
100))
print('\nAverage Score: {:.2f}'.format(average_score))
break
env.close()
logger.close()
timer.finish()
def main():
env_info = env.reset(train_mode=False)[brain_name]
num_agents = len(env_info.agents)
print('Number of agents:', num_agents)
# size of each action
action_size = brain.vector_action_space_size
print('Size of each action:', action_size)
# examine the state space
states = env_info.vector_observations
state_size = states.shape[1]
seeding()
# number of parallel agents
#parallel_envs = num_agents
# number of training episodes.
# change this to higher number to experiment. say 30000.
number_of_episodes = 10000
update_actor_after = 100
update_actor_every = 2
episode_length = 100
batchsize = 100
# how many episodes to save policy and gif
save_interval = 1000
t = 0
LR_ACTOR = 1e-5
LR_CRITIC = 3e-3
# amplitude of OU noise
# this slowly decreases to 0
noise = 1.0
noise_reduction = 0.999999
# how many episodes before update
episode_per_update = 1
no_of_updates_perTime = 1
log_path = os.getcwd() + "/log"
model_dir = os.getcwd() + "/model_dir"
os.makedirs(model_dir, exist_ok=True)
#torch.set_num_threads(parallel_envs)
#env = envs.make_parallel_env(parallel_envs)
# keep 5000 episodes worth of replay
buffer = ReplayBuffer(int(10 * episode_length))
# initialize policy and critic
maddpg = MADDPG(lr_actor=LR_ACTOR, lr_critic=LR_CRITIC)
#logger = SummaryWriter(log_dir=log_path)
agent0_reward = []
agent1_reward = []
#agent2_reward = []
# training loop
# show progressbar
import progressbar as pb
widget = [
'episode: ',
pb.Counter(), '/',
str(number_of_episodes), ' ',
pb.Percentage(), ' ',
pb.ETA(), ' ',
pb.Bar(marker=pb.RotatingMarker()), ' '
]
timer = pb.ProgressBar(widgets=widget, maxval=number_of_episodes).start()
# use keep_awake to keep workspace from disconnecting
for episode in range(0, number_of_episodes):
timer.update(episode)
env_info = env.reset(
train_mode=False)[brain_name] # reset the environment
states = env_info.vector_observations # get the current state (for each agent)
scores = np.zeros(num_agents) # initialize the score (for each agent)
reward_this_episode = np.zeros((1, num_agents))
#all_obs = env.reset() #
obs = states
obs_full = np.concatenate((states[0], states[1]))
#for calculating rewards for this particular episode - addition of all time steps
# save info or not
save_info = ((episode) % save_interval < 1
or episode == number_of_episodes - 1)
tmax = 0
#resetting noise
for i in range(num_agents):
maddpg.maddpg_agent[i].noise.reset()
for episode_t in range(episode_length):
t += 1
update_act = True if (episode > update_actor_after or episode %
update_actor_every == 0) else False
# explore = only explore for a certain number of episodes
# action input needs to be transposed
actions = maddpg.act(transpose_to_tensorAsitis(obs),
noise=noise,
batch=False)
noise *= noise_reduction
actions_array = torch.stack(actions).cpu().detach().numpy()
# transpose the list of list
# flip the first two indices
# input to step requires the first index to correspond to number of parallel agents
actions_for_env = np.rollaxis(actions_array, 1)
# step forward one frame
env_info = env.step(actions_for_env)[brain_name]
next_states = env_info.vector_observations # get next state (for each agent)
rewards = env_info.rewards # get reward (for each agent)
dones = env_info.local_done # see if episode finished
scores += env_info.rewards
rewards_for_env = np.hstack(rewards)
obs = states
obs_full = np.concatenate((states[0], states[1]))
next_obs = next_states
next_obs_full = np.concatenate((next_states[0], next_states[1]))
# add data to buffer
transition = (np.array([obs]), np.array([obs_full]),
np.array([actions_for_env]),
np.array([rewards_for_env]), np.array([next_obs]),
np.array([next_obs_full]),
np.array([dones], dtype='float'))
buffer.push(transition)
reward_this_episode += rewards
obs, obs_full = next_obs, next_obs_full
# update once after every episode_per_update
if len(buffer) > batchsize and episode % episode_per_update == 0:
for _ in range(no_of_updates_perTime):
for a_i in range(num_agents):
samples = buffer.sample(batchsize)
#updating the weights of the n/w
maddpg.update(samples, a_i, update_actor=update_act)
maddpg.update_targets(
) #soft update the target network towards the actual networks
if np.any(dones):
# if the episode is done the loop is break to the next episode
break
for i in range(num_agents):
agent0_reward.append(reward_this_episode[0, 0])
agent1_reward.append(reward_this_episode[0, 1])
if episode % 100 == 0 or episode == number_of_episodes - 1:
avg_rewards = [np.mean(agent0_reward), np.mean(agent1_reward)]
agent0_reward = []
agent1_reward = []
for a_i, avg_rew in enumerate(avg_rewards):
#logger.add_scalar('agent%i/mean_episode_rewards' % a_i, avg_rew, episode)
print('agent%i/mean_episode_rewards' % a_i, avg_rew, episode)
#saving model
save_dict_list = []
if save_info:
for i in range(num_agents):
save_dict = {
'actor_params':
maddpg.maddpg_agent[i].actor.state_dict(),
'actor_optim_params':
maddpg.maddpg_agent[i].actor_optimizer.state_dict(),
'critic_params':
maddpg.maddpg_agent[i].critic.state_dict(),
'critic_optim_params':
maddpg.maddpg_agent[i].critic_optimizer.state_dict()
}
save_dict_list.append(save_dict)
torch.save(
save_dict_list,
os.path.join(model_dir, 'episode-{}.pt'.format(episode)))
# save gif files
#imageio.mimsave(os.path.join(model_dir, 'episode-{}.gif'.format(episode)),
#frames, duration=.04)
timer.finish()
print "Connecting to the database..."
connection = pymysql.connect(host='localhost',
user='******',
password='******',
database='news',
cursorclass=pymysql.cursors.DictCursor)
pbar_cursor = connection.cursor()
pbar_cursor.execute("SELECT count(*) as countdown from `Comment`")
pbar_result = pbar_cursor.fetchone()
pbar_total_rows = pbar_result['countdown']
pbar_widgets = [
'Calculating cat2... ',
progressbar.Counter(), '/{0} '.format(pbar_total_rows),
progressbar.Percentage(), ' ',
progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
progressbar.ETA()
]
cursor = connection.cursor()
print "Sending query..."
select_sql = "SELECT `id`, `Content` FROM `Comment`"
cursor.execute(select_sql)
update_sql = """
UPDATE `Comment`
SET `meanArousal` = %s
,`stdvArousal` = %s
,`meanValence` = %s
,`stdvValence` = %s
WHERE `id` = %s
def train(self,
num_episodes: int = 100,
max_t: int = None,
add_noise: bool = True,
scores_window_size: int = 100,
save_every: int = None) -> List[float]:
"""Trains agent(s) through interaction with this environment.
Args:
num_episodes (int): Number of episodes to train the agent for.
max_t (int): Maximum number of timesteps in an episode.
add_noise (boolean): Add noise to actions.
scores_window_size (int): Window size for average score display.
save_every (int): Save state dicts every `save_every` episodes.
Returns:
The scores for each episode.
"""
widget = [
"Episode: ",
pb.Counter(), '/',
str(num_episodes), ' ',
pb.Percentage(), ' ',
pb.ETA(), ' ',
pb.Bar(marker=pb.RotatingMarker()), ' ', 'Rolling Average: ',
pb.FormatLabel('')
]
timer = pb.ProgressBar(widgets=widget, maxval=num_episodes).start()
self.start_env()
self.episode_scores = []
for i_episode in range(1, num_episodes + 1):
current_average = self.get_current_average_score(
scores_window_size)
widget[12] = pb.FormatLabel(str(current_average)[:6])
timer.update(i_episode)
save_info = save_every and i_episode % save_every == 0
observation = self.env.reset()
observation = self.normalize_observation(observation)
scores = np.zeros(self.num_agents)
rewards = []
self.algorithm.reset()
frames = []
if save_info:
frames.append(self.env.render("rgb_array"))
t = 1
while True:
if max_t and t == max_t + 1:
break
action = self.act(observation,
add_noise=add_noise,
logger=self.logger)
next_observation, reward, done, _ = self.env.step(action)
next_observation = self.normalize_observation(next_observation)
self.algorithm.step(observation,
action,
reward,
next_observation,
done,
logger=self.logger)
observation = next_observation
scores += reward
rewards.append(reward)
t += 1
if save_info:
frames.append(self.env.render("rgb_array"))
if done:
break
self.episode_scores.append(scores)
self.algorithm.update(rewards, logger=self.logger)
if save_info:
# TODO: Only save if best weights so far
self.algorithm.save_state_dicts()
if self.logger:
self.logger.add_scalar("avg_rewards", np.mean(rewards),
i_episode)
if self.gifs_recorder:
self.gifs_recorder.save_gif(
"episode-{}.gif".format(i_episode), frames)
self.close_env()
if self.logger:
self.logger.close()
return self.episode_scores
# -- Simulation configuration I: parsing, debugging.
conf_file, debug, args1, hlp = parser_init()
if not hlp:
logger = log_conf(debug)
else:
logger = None
# -- Simulation configuration II: data entry (second parser).
description = 'Conductance based QIF spiking neural network. All to all coupled with distributed external currents.'
opts, args = parser(conf_file, args1, description=description) # opts is a dictionary, args is an object
# Parameters are now those introduced in the configuration file:
# >>> args.parameter1 + args.parameter2
d = Data(args.N, args.e, args.g, args.E, args.d, args.t0, args.tf, args.dt, 1.0, args.tm, args.D, args.s)
fr = FiringRate(data=d, swindow=0.1, sampling=0.05)
# Progress-bar configuration
widgets = ['Progress: ', pb.Percentage(), ' ',
pb.Bar(marker=pb.RotatingMarker()), ' ', pb.ETA(), ' ']
# ############################################################
# 0) Prepare simulation environment
pbar = pb.ProgressBar(widgets=widgets, maxval=10 * (d.nsteps + 1)).start()
time1 = timer()
tstep = 0
temps = 0
kp = k = 0
# Time loop
while temps < d.tfinal:
# TIme step variables
kp = tstep % d.nsteps
k = (tstep + d.nsteps - 1) % d.nsteps
k2p = tstep % 2
k2 = (tstep + 2 - 1) % 2
widgets = [
' [',
progressbar.Timer(), '] ',
progressbar.Bar(),
progressbar.Percentage(), ' (',
progressbar.ETA(), ') ', '[',
progressbar.Counter(), ']'
]
widgets_unknown = [
' [',
progressbar.Timer(),
'] ',
progressbar.RotatingMarker(),
' (',
progressbar.Counter(),
') ',
]
# print("Reading Filters from [{}] ...".format(FILTER_PATTERNS_FILE ))
# with open(FILTER_PATTERNS_FILE, "r",) as pattern_file:
# for line in pattern_file:
# if s := line.strip():
# if not s.startswith("//#"):
# FILTERS.append(s)
def yes_or_no(question):
reply = str(input('{} (Y/n): '.format(question))).lower().strip()
def __import_records(self,
records,
user_id,
errors,
show_progressbar=False,
batch_size=100):
domain_mapping = self.__get_domain_mapping(user_id)
widget = [
progressbar.FormatLabel(''), ' ',
progressbar.Percentage(), ' ',
progressbar.Bar('#'), ' ',
progressbar.RotatingMarker(), ' ',
progressbar.ETA()
]
if show_progressbar:
widget[0] = progressbar.FormatLabel('Importing records')
bar = progressbar.ProgressBar(max_value=len(records),
widgets=widget)
count = 0
for record_to_import in records:
count += 1
bar.update(count) if show_progressbar else False
# First, get the zone.
zone_id = domain_mapping[
record_to_import['domain']] if record_to_import[
'domain'] in domain_mapping else None
if not zone_id:
# At this point all zones should exist.
errors.append('Could not find zone: {0}'.format(
record_to_import['domain']))
continue
data = json.dumps(record_to_import['data']) if isinstance(
record_to_import['data'], dict) else record_to_import['data']
conditional_data = json.dumps(
record_to_import['conditional_data']) if isinstance(
record_to_import['conditional_data'],
dict) else record_to_import['conditional_data']
self.__record_update_or_create(
zone_id,
record_to_import['ttl'],
record_to_import['cls'],
record_to_import['type'],
record_to_import['active'],
data,
record_to_import['is_conditional'],
record_to_import['conditional_count'],
record_to_import['conditional_limit'],
record_to_import['conditional_reset'],
conditional_data,
id=record_to_import['record_id'],
autocommit=False)
if count % batch_size == 0:
db.session.commit()
db.session.commit()
return True
def process_loader(name, loader, output_path):
# recreate output path
if os.path.isdir(output_path):
shutil.rmtree(output_path)
os.makedirs(output_path)
# get trains and tests
trains = loader.collect_train_list()
tests = loader.collect_test_list()
# process trains
def process(f):
# get label path
label_path = os.path.join(output_path, f)
label_path = os.path.splitext(label_path)[0] + '.yml'
label_dir = os.path.dirname(label_path)
if not os.path.isdir(label_dir): os.makedirs(label_dir)
# init doc
doc = Document()
doc.dataset = name
doc.path = f
doc.seg_tag = False
doc.box_tag = False
# process
loader.process(f, doc)
# set tag
doc.seg_tag = doc.search('segmentation_id') is not None
doc.box_tag = doc.search('box') is not None
# save
doc.save(label_path)
return doc
train_docs = []
test_docs = []
widgets = [
progressbar.FormatLabel(name), ' ',
progressbar.Percentage(), ' ',
progressbar.Bar('#'), ' ',
progressbar.RotatingMarker()
]
with progressbar.ProgressBar(max_value=len(trains) + len(tests),
widgets=widgets) as bar:
index = 0
for i in range(len(trains)):
train_docs.append(process(trains[i]))
bar.update(index)
index += 1
for i in range(len(tests)):
test_docs.append(process(tests[i]))
bar.update(index)
index += 1
# output file list
doc = Document()
doc.trains = [os.path.splitext(p)[0] + '.yml' for p in trains]
doc.tests = [os.path.splitext(p)[0] + '.yml' for p in tests]
doc.save(os.path.join(output_path, name) + ".yml")
def __process_records(self, records, user, show_progressbar=False):
errors = []
items = []
widget = [
progressbar.FormatLabel(''), ' ',
progressbar.Percentage(), ' ',
progressbar.Bar('#'), ' ',
progressbar.RotatingMarker(), ' ',
progressbar.ETA()
]
if show_progressbar:
widget[0] = progressbar.FormatLabel('Processing records')
bar = progressbar.ProgressBar(max_value=len(records),
widgets=widget)
domain_mapping = self.__get_domain_mapping(user.id)
domain_mapping_reverse = self.__get_domain_mapping(user.id,
reverse=True)
count = 0
for record in records:
count += 1
bar.update(count) if show_progressbar else False
record_errors = []
active = True if record['active'] in ['1', 'yes', 'true'
] else False
zone_id = self.__process_record_zone(record, record_errors,
domain_mapping)
record_id = self.__process_record_id(record, zone_id,
record_errors,
domain_mapping_reverse)
ttl = self.__process_record_ttl(record, record_errors)
cls = self.__process_record_cls(record, record_errors)
type = self.__process_record_type(record, record_errors)
is_conditional = True if record['is_conditional'] in [
'1', 'yes', 'true'
] else False
conditional_reset = True if record['conditional_reset'] in [
'1', 'yes', 'true'
] else False
conditional_count = self.__process_number(record, record_errors,
'conditional_count')
conditional_limit = self.__process_number(record, record_errors,
'conditional_limit')
data = {}
conditional_data = {}
if len(type) > 0:
data = self.__process_record_data(record, type, record_errors)
if is_conditional:
conditional_data = self.__process_record_data(
record, type, record_errors, is_conditional=True)
if len(record_errors) == 0:
items.append({
'record_id': record_id,
'zone_id': zone_id,
'domain': record['domain'],
'active': active,
'ttl': ttl,
'cls': cls,
'type': type,
'data': data,
'is_conditional': is_conditional,
'conditional_count': conditional_count,
'conditional_limit': conditional_limit,
'conditional_reset': conditional_reset,
'conditional_data': conditional_data
})
else:
errors += record_errors
return items, errors
def StartProgressBar(maxValue):
''' StartProgressBar(maxValue):
Simple module to create and initialise a progress bar. The argument "maxvalue" refers to the
total number of tasks to be completed. This must be defined at the start of the progress bar.
'''
import progressbar
widgets = [progressbar.FormatLabel(''), ' ', progressbar.Percentage(), ' ', progressbar.Bar('/'), ' ', progressbar.RotatingMarker()]
pBar = progressbar.ProgressBar(widgets=widgets, maxval=maxValue)
if pBar.start_time is None:
pBar.start()
return pBar
def _analyze(self):
"""
Perform a full code scan on the target binary, and try to identify as much code as possible.
In order to identify as many functions as possible, and as accurate as possible, the following operation
sequence is followed:
# Active scanning
- If the binary has "function symbols" (TODO: this term is not accurate enough), they are starting points of
the code scanning
- If the binary does not have any "function symbol", we will first perform a function prologue scanning on the
entire binary, and start from those places that look like function beginnings
- Otherwise, the binary's entry point will be the starting point for scanning
# Passive scanning
- After all active scans are done, we will go through the whole image and scan all code pieces
"""
# We gotta time this function
start_time = datetime.now()
traced_address = set()
self.graph = networkx.DiGraph()
initial_state = self.project.factory.blank_state(mode="fastpath")
initial_options = initial_state.options - {
simuvex.o.TRACK_CONSTRAINTS
} - simuvex.o.refs
initial_options |= {simuvex.o.SUPER_FASTPATH}
# initial_options.remove(simuvex.o.COW_STATES)
initial_state.options = initial_options
# Sadly, not all calls to functions are explicitly made by call
# instruction - they could be a jmp or b, or something else. So we
# should record all exits from a single function, and then add
# necessary calling edges in our call map during the post-processing
# phase.
function_exits = defaultdict(set)
widgets = [
progressbar.Percentage(), ' ',
progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
progressbar.Timer(), ' ',
progressbar.ETA()
]
pb = progressbar.ProgressBar(widgets=widgets,
maxval=10000 * 100).start()
starting_points = set()
rebase_addr = self._binary.rebase_addr
if self._use_symbols:
starting_points |= set([
addr + rebase_addr for addr in self._func_addrs_from_symbols()
])
if self._use_function_prologues:
starting_points |= set([
addr + rebase_addr
for addr in self._func_addrs_from_prologues()
])
starting_points = sorted(list(starting_points), reverse=True)
while True:
maybe_function = False
if starting_points:
next_addr = starting_points.pop()
maybe_function = True
if self._seg_list.is_occupied(next_addr):
continue
else:
next_addr = self._next_code_addr(initial_state)
percentage = self._seg_list.occupied_size * 100.0 / self._valid_memory_region_size
if percentage > 100.0:
percentage = 100.0
pb.update(percentage * 10000)
if next_addr is not None:
l.info("Analyzing %xh, progress %0.04f%%", next_addr,
percentage)
else:
l.info('No more addr to analyze. Progress %0.04f%%',
percentage)
break
self._scan_code(traced_address, function_exits, initial_state,
next_addr, maybe_function)
pb.finish()
end_time = datetime.now()
l.info("A full code scan takes %d seconds.",
(end_time - start_time).seconds)
def compare_graph(self, bedgraph, fileout):
"""************************************************************************************************************************************************************
Task: compares the scores of this bedgraph file with the scores in another bedgraph file.
Inputs:
bedgraph: bedgraph_file object representing the other bedgraph file.
fileout: string containing the full path to the png file were the comparative graph will be saved.
Ouputs: a new png file will be created named fileout.
************************************************************************************************************************************************************"""
starts1 = {}
ends1 = {}
scores1 = {}
fd = file(self.filename)
fd.readline(); fd.readline(); fd.readline()
# Loads chr,start,end,score from each line
for i,line in enumerate(fd):
fields = line.split('\t')
# Check whether there is already an entry for current chromosome in starts1
if(fields[0] not in starts1):
try:
scores1[fields[0]] = [float(fields[3])]
starts1[fields[0]] = [float(fields[1])]
ends1[fields[0]] = [float(fields[2])]
except ValueError:
print 'WARNING: '+fields[3]+' at line '+str(i+4)+' of file '+self.filename+' is not a number.'
else:
try:
scores1[fields[0]].append(float(fields[3]))
starts1[fields[0]].append(float(fields[1]))
ends1[fields[0]].append(float(fields[2]))
except ValueError:
print 'WARNING: '+fields[3]+' at line '+str(i+4)+' of file '+self.filename+' is not a number.'
fd.close()
starts2 = {}
ends2 = {}
scores2 = {}
fd = file(bedgraph.filename)
fd.readline(); fd.readline(); fd.readline()
# Loads chr,start,end,score from each line
for i,line in enumerate(fd):
fields = line.split('\t')
# Check whether there is already an entry for current chromosome in starts2
if(fields[0] not in starts2):
starts2[fields[0]] = [float(fields[1])]
ends2[fields[0]] = [float(fields[2])]
try:
scores2[fields[0]] = [float(fields[3])]
except ValueError:
print 'WARNING: '+fields[3]+' at line '+str(i+4)+' of file '+bedgraph.filename+' is not a number.'
else:
starts2[fields[0]].append(float(fields[1]))
ends2[fields[0]].append(float(fields[2]))
try:
scores2[fields[0]].append(float(fields[3]))
except ValueError:
print 'WARNING: '+fields[3]+' at line '+str(i+4)+' of file '+bedgraph.filename+' is not a number.'
fd.close()
# Transform the lists of each chromosome in a numpy.array
for chr in starts1:
starts1[chr] = numpy.array(starts1[chr])
ends1[chr] = numpy.array(ends1[chr])
# Transform the lists of each chromosome in a numpy.array
for chr in starts2:
starts2[chr] = numpy.array(starts2[chr])
ends2[chr] = numpy.array(ends2[chr])
widgets = ['Comparing windows: ', progressbar.Percentage(), ' ',
progressbar.Bar(marker=progressbar.RotatingMarker()), ' ', progressbar.ETA()]
pbar = progressbar.ProgressBar(widgets=widgets, maxval=len(starts1)).start()
exactcount = 0
x=[]; y=[]
# Compare regions in each chromosome independently
for k,chr in enumerate(starts1):
# Check that there are regions at current chromosome in 'bedgraph'
if(chr in starts2):
# For each region of current chromosome in self, look for overlaps in bedgraph
for i in range(len(starts1[chr])):
# Check whether any region in bedgraph overlaps with the 'start' limit of current region
overlap = ((starts1[chr][i]>=starts2[chr]) * (starts1[chr][i]<=ends2[chr])).nonzero()[0]
if(len(overlap)>0):
j = 0
# Check whether any of the overlapping regions is actually the same exact region
while(j<len(overlap) and (starts1[chr][i]<>starts2[chr][overlap[j]] or ends1[chr][i]<>ends2[chr][overlap[j]])):
j += 1
# Check whether an exact match was found in the while above
if(j<len(overlap)):
exactcount += 1
x.append(scores1[chr][i])
y.append(scores2[chr][overlap[j]])
else:
x.append(scores1[chr][i])
y.append(scores2[chr][overlap[0]])
else:
# Check whether any region in bedgraph overlaps with the 'end' limit of current region
overlap = ((ends1[chr][i]>=starts2[chr]) * (ends1[chr][i]<=ends2[chr])).nonzero()[0]
if(len(overlap)>0):
x.append(scores1[chr][i])
y.append(scores2[chr][overlap[0]])
pbar.update(k+1)
pbar.finish()
print str(len(x))+' windows overlap between both tracks'
print str(exactcount)+' windows perfectly match between both tracks'
fig = pyplot.figure(figsize=(13,10))
ax = fig.add_subplot(111)
ax.scatter(x,y)
ax.set_ylabel(os.path.basename(bedgraph.filename))
ax.set_xlabel(os.path.basename(self.filename))
# fig = pyplot.figure()
# ax = fig.add_subplot(111, projection='3d')
# hist, xedges, yedges = numpy.histogram2d(x, y, bins=4)
#
# elements = (len(xedges) - 1) * (len(yedges) - 1)
# xpos, ypos = numpy.meshgrid(xedges+0.25, yedges+0.25)
#
# xpos = xpos.flatten()
# ypos = ypos.flatten()
# zpos = numpy.zeros(elements)
# dx = 0.5 * numpy.ones_like(zpos)
# dy = dx.copy()
# dz = hist.flatten()
#
# ax.bar3d(xpos, ypos, zpos, dx, dy, dz, color='b', zsort='average')
#
# ax.set_ylabel(os.path.basename(bedgraph.filename))
# ax.set_xlabel(os.path.basename(self.filename))
# ax.bar3d(dx, dy, dz, color='b', zsort='average')
fig.savefig(fileout)
matplotlib.pyplot.close(fig)
