def __init__(self, *args, **kwargs):
"""Initialization
Args:
num_threads(int):
number of threads, defaults to number of cores
that are available on system
"""
super(AbstractLocalThreadPool, self).__init__(*args, **kwargs)
# Store tasks
self.tasks = Queue()
#Threads put themselves here when they die
self.dead_thread_queue = Queue()
self.shutdown_event = Event()
self.shutdown_event.clear()
self.WorkerCheckInterval=kwargs.get('WorkerCheckInterval', 0.5)
self._max_threads = kwargs.get('num_threads',
multiprocessing.cpu_count())
if self._max_threads is None:
self._max_threads = multiprocessing.cpu_count()
self.keep_alive_thread = None
self.threads = []
def run(args):
logging.basicConfig(level=int(round(10*args.verbose_level)))
assert args.n_core <= multiprocessing.cpu_count(), 'Requested n_core={} > cpu_count={}'.format(
args.n_core, multiprocessing.cpu_count())
def Start():
LOG.info('Started a worker in {} from parent {}'.format(
os.getpid(), os.getppid()))
exe_pool = Pool(args.n_core, initializer=Start)
if args.trim:
get_consensus = get_consensus_with_trim
else:
get_consensus = get_consensus_without_trim
K = 8
config = args.min_cov, K, \
args.max_n_read, args.min_idt, args.edge_tolerance, args.trim_size, args.min_cov_aln, args.max_cov_aln
# TODO: pass config object, not tuple, so we can add fields
inputs = []
for datum in get_seq_data(config, args.min_n_read, args.min_len_aln):
inputs.append((get_consensus, datum))
try:
LOG.info('running {!r}'.format(get_consensus))
for res in exe_pool.imap(io.run_func, inputs):
process_get_consensus_result(res, args)
LOG.info('finished {!r}'.format(get_consensus))
except:
LOG.exception('failed gen_consensus')
exe_pool.terminate()
raise
def input_fn():
def decode(elem):
model_features = tf.parse_single_example(elem, features=feature_spec)
model_labels = tf.stack([model_features.pop(label) for label in labels])
return model_features, model_labels
# For more information, check:
# https://www.tensorflow.org/performance/datasets_performance
files = tf.data.Dataset.list_files(file_pattern)
dataset = files.apply(tf.contrib.data.parallel_interleave(
tf.data.TFRecordDataset, cycle_length=mp.cpu_count()))
dataset = dataset.map(decode, num_parallel_calls=mp.cpu_count())
dataset = dataset.take(-1)
if mode == tf.estimator.ModeKeys.TRAIN:
if shuffle:
dataset = dataset.apply(tf.contrib.data.shuffle_and_repeat(
batch_size * 8))
else:
dataset = dataset.cache()
dataset = dataset.repeat()
dataset = dataset.batch(batch_size)
dataset = dataset.prefetch(1)
iterator = dataset.make_one_shot_iterator()
batch_features, batch_labels = iterator.get_next()
return batch_features, batch_labels
def main(treeModelPath, dataInputPath, resultOutPath, debug):
# read model
treeModel = readModel(treeModelPath)
# create output dir
if not os.path.isdir(resultOutPath):
os.mkdir(resultOutPath)
if debug:
pool = multiprocessing.Pool(processes=1)
else:
pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
print "Number of core: %d" % (multiprocessing.cpu_count())
start_time = datetime.now()
jobN = 0
for filename in os.listdir(dataInputPath):
if ".json" in filename:
if debug:
# debug model just test 1 file in 1 process
# filterFiles(jobN,filename,treeModel,debug)
pool.apply_async(filterFiles, (jobN, filename, dataInputPath, resultOutPath, treeModel, debug))
break
else:
pool.apply_async(filterFiles, (jobN, filename, dataInputPath, resultOutPath, treeModel, debug))
jobN += 1
pool.close()
pool.join()
diff = datetime.now() - start_time
print "Spend %d.%d seconds" % (diff.seconds, diff.microseconds)
def get_needle_tips(images):
"""Get sample tips from images."""
tips = []
results = []
# Do not make more processes than needed for the number of images.
if len(images) > multiprocessing.cpu_count():
proc_count = multiprocessing.cpu_count()
else:
proc_count = len(images)
pool = Pool(processes=proc_count)
for image in images:
results.append(pool.apply_async(_get_ellipse_point,
args=(image,)))
for result in results:
tip = result.get()
if tip is not None:
tips.append(tip)
if len(tips) == 0:
raise ValueError("No sample tip points found.")
return tips
def sample_paths_parallel(N,
policy,
baseline,
env_mode='train',
T=1e6, gamma=1,
num_cpu=None,
max_process_time=60,
max_timeouts=5,
normalized_env=False):
if num_cpu == None or num_cpu == 'max':
num_cpu = mp.cpu_count()
elif num_cpu == 1:
return sample_paths(N, policy, baseline, evn_mode, T, gamma, normalized_env)
else:
num_cpu = min(mp.cpu_count(), num_cpu)
paths_per_cpu = int(np.ceil(N/num_cpu))
args_list = [paths_per_cpu, policy, baseline, env_mode, T, gamma, normalized_env]
results = _try_multiprocess(args_list, num_cpu, max_process_time, max_timeouts)
paths = []
# result is a paths type and results is list of paths
for result in results:
for path in result:
paths.append(path)
return paths
def Nbeam(width, height, start_board, max_turn, playnum, parms):
if mp.cpu_count() == 1:
use_cpu_count = 1
else:
use_cpu_count = mp.cpu_count() - 1
#use_cpu_count = 1
#print "use cpu count: " + str(use_cpu_count)
p = mp.Pool(use_cpu_count)
func_args = []
for i in range(width*height):
func_args.append((search_node_array, width, height, max_turn, playnum, parms, i, start_board))
node_array = p.map(wrap_search_node_array, func_args)
node_array = list(itertools.chain.from_iterable(node_array))
idx = 0
best = 0
for k,v in enumerate(node_array):
if best < v.score:
best = v.score
idx = k
#print "best score:" + str(node_array[idx].score)
#print "best combo:" + str(node_array[idx].combo)
return node_array[idx]
def multiprocess(function, iterator, func_args=None, combine=True, n_pool=-1, include_process_num=True):
"""
Processes data on n number of processors, use n_pool = -1 for all processors
function is the function to apply to each object in iterator, must except the object to process and process number if include_process_num=True
func_args is a dictionary of args to feed function
iterator must be list of files or chunks of dataframes
if combine is True will return the combined data
n_pool can be greater than 1 and less than the cpu count or if negative is 1+cpu_count-|n_pool|
"""
# Initialize pool
if n_pool < 0:
pool = mp.Pool(processes=mp.cpu_count()+1+n_pool)
elif n_pool > 1 and n_pool <= mp.cpu_count():
pool = mp.Pool(processes=n_pool)
else:
raise ValueError("n_pool is out of range for cpu count!")
# Apply Async (returns process once complete
if include_process_num == True:
results = [pool.apply_async(function, args=(obj, process), kwds=func_args) for process, obj in enumerate(iterator)]
else:
results = [pool.apply_async(function, args=(obj), kwds=func_args) for obj in iterator]
# Get results and concat into a single dataframe
if combine == True:
result = pd.concat([p.get() for p in results])
else:
result = [p.get() for p in results]
return result
def profile_locks(cmd):
(c_samples, c, cc, c_dev, sections) = lttng_session( "profile_c"
, cmd(1, 'c')
, ['memcached:c_begin', 'memcached:c_end', 'memcached:inside_cc']
, measure_c)
n = lttng_session( "profile_n"
, cmd(multiprocessing.cpu_count(),'c')
, ['memcached:contention']
, measure_n)[1]
(nsamples, blk_samples) = lttng_session( "profile_block_costs"
, cmd(1,'c')
, ['memcached:block_id']
, measure_blocks)
blk_cnts = lttng_session( "profile_block_counts"
, cmd(1,'c')
, ['memcached:blk_cnts']
, count_blocks)
blk_costs = {k: ((blk_samples[k] * c / nsamples), v) for k, v in blk_cnts.items() if k in blk_samples}
nn = lttng_session( "profile_contention"
, cmd(multiprocessing.cpu_count(),'f')
, ['memcached:contention']
, measure_n)[1]
return((c,c_dev),cc,n,nn,sections,blk_costs)
def buildTFIDFDictionary(csvName):
things = csv_object.getThings(csvName)
global words
global descriptions
for thingy in things:
description = thingy.description
descriptions.append(description)
wordInDoc = description
words = words.union(wordInDoc)
# dictionary = [["None" for x in range(len(things)+1)]"None " for x in range(len(words)+1)]] #define matrix of things and words
# dictionary = {}
# multiprocessing
print cpu_count(), len(things)
thingPool = Pool(cpu_count())
results = thingPool.map(thingThreadHelper, things)
# for i, thingy in enumerate(things):
# dictionary[i] = thing.title
# for j, word in enumerate(words):
# dictionary[i][0]= word
# dictionary[i][j] = tfidf(word, thingy.description, descriptions)))
# print type(results)
# print results
# for r in results:
# print r
return results
def parse_arguments():
"""
Encapsulate the use of argparse
@param: None
@return: An instance of argparse
"""
parser = argparse.ArgumentParser(description="Create some random charts")
# Required
# Nothing
# Optional
parser.add_argument("-e", "--executable",
help="The executable to use [default: None]",
type=str,
default=None)
parser.add_argument("-n", "--number",
help="Number of charts to randomly create [default: %(default)s]",
type=check_positive,
default=1)
parser.add_argument("-t", "--threads",
help="Number of threads to use [default: %(default)s]",
type=int,
default=multiprocessing.cpu_count()-1,
choices=range(1, multiprocessing.cpu_count()))
return parser.parse_args()
def count_free_cores(max=cpu_count()):
"""Count the number of CPU cores not currently used by this job.
"""
if max is True:
max = cpu_count()
active = 1 #len(active_children()
return max - (active + 1)
def GetParallelProcessCount(self):
# limit based on free memory
f = os.popen('vmstat', 'r')
f.readline()
f.readline()
line = f.readline()
f.close()
freeRAM = line.split()[3]
cache = line.split()[5]
ppCount = int((float(freeRAM) + float(cache)) / 80000.0)
if ppCount > multiprocessing.cpu_count(): # *three* extra processes
ppCount = multiprocessing.cpu_count()
if ppCount < 1: # need at least one process
ppCount = 1
# now limit based on CPU load
f = open('/proc/loadavg', 'r')
line = f.readline()
f.close()
load = float(line.split()[0])
if load > (float(multiprocessing.cpu_count()) + 0.5) and ppCount > 3:
ppCount = 3
if load > (float(multiprocessing.cpu_count()) + 1.0) and ppCount > 2:
ppCount = 2
if load > (float(multiprocessing.cpu_count()) + 1.5) and ppCount > 1:
ppCount = 1
return ppCount
def main(opts):
"""The main loop of the module, do the renaming in parallel etc."""
log = logging.getLogger("exif2timestream")
setup_logs(opts)
# beginneth the actual main loop
start_time = time()
cameras = parse_camera_config_csv(opts["-c"])
n_images = 0
for camera in cameras:
msg = "Processing experiment {}, location {}\n".format(
camera[FIELDS["expt"]],
camera[FIELDS["location"]],
)
msg += "Images are coming from {}, being put in {}".format(
camera[FIELDS["source"]],
camera[FIELDS["destination"]],
)
print(msg)
log.info(msg)
for ext, images in find_image_files(camera).iteritems():
images = sorted(images)
n_cam_images = len(images)
print("{0} {1} images from this camera".format(n_cam_images, ext))
log.info("Have {0} {1} images from this camera".format(
n_cam_images, ext))
n_images += n_cam_images
last_date = None
subsec = 0
count = 0
# TODO: sort out the whole subsecond clusterfuck
if "-1" in opts and opts["-1"]:
log.info("Using 1 process (What is this? F*****g 1990?)")
for image in images:
count += 1
print("Processed {: 5d} Images".format(count), end='\r')
process_image((image, camera, ext))
else:
from multiprocessing import Pool, cpu_count
if "-t" in opts and opts["-t"] is not None:
try:
threads = int(opts["-t"])
except ValueError:
threads = cpu_count() - 1
else:
threads = cpu_count() - 1
# Ensure that we're using at least one thread
threads = max(threads, 1)
log.info("Using {0:d} processes".format(threads))
# set the function's camera-wide arguments
args = zip(images, cycle([camera]), cycle([ext]))
pool = Pool(threads)
for _ in pool.imap(process_image, args):
count += 1
print("Processed {: 5d} Images".format(count), end='\r')
pool.close()
pool.join()
print("Processed {: 5d} Images. Finished this cam!".format(count))
secs_taken = time() - start_time
print("\nProcessed a total of {0} images in {1:.2f} seconds".format(
n_images, secs_taken))
def ProcessStuff(spp_list):
print 'cpu_count() = %d\n' % multiprocessing.cpu_count()
NUMBER_OF_PROCESSES = multiprocessing.cpu_count()
TASKS = [(CallMaxent, (spp_list[i],)) for i in range(len(spp_list))]
#TASKS2 = [(plus, (i, 8)) for i in range(10)]
# Create queues
task_queue = Queue()
done_queue = Queue()
# Submit tasks
for task in TASKS:
task_queue.put(task)
# Start worker processes
for i in range(NUMBER_OF_PROCESSES):
Process(target=worker, args=(task_queue, done_queue)).start()
# Get and print results
print 'Unordered results:'
for i in range(len(TASKS)):
print '\t', done_queue.get()
# Tell child processes to stop
for i in range(NUMBER_OF_PROCESSES):
task_queue.put('STOP')
def main(dataInputPath,resultOutPath,ptnOutputPath):
model, table = projizz.readPrefixTreeModel("../prefix_tree_model/patternTree.json")
if not os.path.isdir(resultOutPath):
os.mkdir(resultOutPath)
if not os.path.isdir(ptnOutputPath):
os.mkdir(ptnOutputPath)
pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
print "Number of core: %d" % (multiprocessing.cpu_count())
start_time = datetime.now()
jobN = 0
for filename in os.listdir(dataInputPath):
if ".json" in filename:
pool.apply_async(tryToFindRela, (jobN, filename, dataInputPath, resultOutPath,ptnOutputPath, model, table))
jobN+=1
pool.close()
pool.join()
diff = datetime.now() - start_time
print "Spend %d.%d seconds" % (diff.seconds,diff.microseconds)
projizz.combinedFileWriter(model,os.path.join(ptnOutputPath,"model"))
projizz.combinedFileWriter(table,os.path.join(ptnOutputPath,"table"))
def __init__(self, tasks, num_cpus=0, profiling=False):
self.log = LoggingManager.get_logger('kraken')
try:
self.num_processes = int(num_cpus)
if self.num_processes < 1:
raise ValueError()
if self.num_processes > cpu_count():
self.log.warning("Number of cores (%d) larger than available." % self.num_processes)
raise ValueError()
except (ValueError, TypeError):
self.log.warning("Number of cores has not been specified or is incorrect. Using available cores.")
self.num_processes = cpu_count()
self.log.info("Kraken has %d tentacles (cpu cores)" % self.num_processes)
self.tasks = tasks
self.num_tasks = len(tasks)
self.tentacles = []
tentacle_tasks = [tasks[i::self.num_processes] for i in xrange(self.num_processes)]
for i in range(self.num_processes):
tentacle = Tentacle(tentacle_tasks[i], profiling)
self.tentacles.append(tentacle)
self.log.info("%d ships ready to be smashed" % self.num_tasks)
def interface():
'''Command-line interface'''
usage = "usage: %prog [options]"
p = optparse.OptionParser(usage)
p.add_option('--output', dest = 'output', action='store', type='string',
default = None, help='The path to the file where you want to store the '+
'barcodes', metavar='FILE')
p.add_option('--tag-length', dest = 'tl', action='store',
type='int', default = 6, help='The desired tag length')
p.add_option('--edit-distance', dest = 'ed', action='store',
type='int', default = 3, help='The desired edit distance')
p.add_option('--multiprocessing', dest = 'multiprocessing',
action='store_true', default=False, help='Use multiprocessing')
p.add_option('--processors', dest = 'nprocs', action='store',
type='int', default = None, help='The number of processing cores to use when' +
' using multiprocessing. Default is # of cores - 2')
p.add_option('--no-polybase', dest = 'polybase', action='store_true', default=False,
help='Remove tags with > 2 identical nucleotides in a row')
p.add_option('--gc', dest = 'gc', action='store_true', default=False,
help='Remove tags with GC content (%) 40 > x > 60')
p.add_option('--comp', dest = 'comp', action='store_true', default=False,
help='Remove tags that are perfect self-complements')
p.add_option('--hamming', dest = 'hamming', action='store_true', default=False,
help='Use Hamming distance in place of edit (Levenshtein) distance.')
# c is on by default now
#p.add_option('--use-c', dest = 'clev', action='store_true', default=False,
#help='Use the C version of Levenshtein (faster)')
p.add_option('--min-and-greater', dest = 'greater', action='store_true', default=False,
help='Show tags at all integer values of edit distance > that specified')
p.add_option('--rescan', dest = 'rescan', action='store', type='string',
default = None, help='Rescan a file')
p.add_option('--rescan-length', dest = 'rescan_length', action='store', type='int',
default = 6, help='Rescan length')
(options,arg) = p.parse_args()
assert options.nprocs <= multiprocessing.cpu_count(), \
"Processors count must equal those available"
# set the number of processors by default
if options.multiprocessing and not options.nprocs:
options.nprocs = multiprocessing.cpu_count() - 2
if not options.tl:
p.print_help()
sys.exit(2)
return options, arg
def parse_opt(self):
""" parses the command line options for different settings. """
optparser = optparse.OptionParser()
optparser.add_option('-c', '--config',
action='store', dest='config', type='string', default='experiments.cfg',
help="your experiments config file")
optparser.add_option('-n', '--numcores',
action='store', dest='ncores', type='int', default=cpu_count(),
help="number of processes you want to use, default is %i"%cpu_count())
optparser.add_option('-d', '--del',
action='store_true', dest='delete', default=False,
help="delete experiment folder if it exists")
optparser.add_option('-e', '--experiment',
action='append', dest='experiments', type='string',
help="run only selected experiments, by default run all experiments in config file.")
optparser.add_option('-b', '--browse',
action='store_true', dest='browse', default=False,
help="browse existing experiments.")
optparser.add_option('-B', '--Browse',
action='store_true', dest='browse_big', default=False,
help="browse existing experiments, more verbose than -b")
optparser.add_option('-p', '--progress',
action='store_true', dest='progress', default=False,
help="like browse, but only shows name and progress bar")
options, args = optparser.parse_args()
self.options = options
return options, args
def run_calculation():
""" Begin multi-process calculation, and save to file """
print("Creating %d-process pool" % mp.cpu_count())
pool = mp.Pool(mp.cpu_count())
f = h5py.File('mandelbrot.hdf5','w')
print("Creating output dataset with shape %s x %s" % (NX, NY))
dset = f.create_dataset('mandelbrot', (NX,NY), 'i')
dset.attrs['XSTART'] = XSTART
dset.attrs['YSTART'] = YSTART
dset.attrs['XEXTENT'] = XEXTENT
dset.attrs['YEXTENT'] = YEXTENT
result = pool.imap(compute_row, (x*xincr for x in xrange(NX)))
for idx, arr in enumerate(result):
if idx%25 == 0: print("Recording row %s" % idx)
dset[idx] = arr
print("Closing HDF5 file")
f.close()
print("Shutting down process pool")
pool.close()
pool.join()
def set_performance(value, notify):
"""Set performance mode.
This will set CPU frequency scaling as well as the
Asus Super Hybrid Engine (FSB speed).
Keyword arguments:
value -- "performance", "ondemand" or "powersave" (str)
notify -- Show popup notification (bool)
"""
modes = {"performance": '0', "ondemand": '1', "powersave": '2'}
try:
with open("/sys/devices/platform/eeepc-wmi/cpufv", 'w') as f:
f.write(modes[value])
except:
pass
for cpu in range(0, multiprocessing.cpu_count()):
with open(
"/sys/devices/system/cpu/cpu{0}/cpufreq/scaling_governor".format(
cpu), 'w') as f:
f.write(value)
with open(
"/sys/devices/system/cpu/cpu{0}/cpufreq/scaling_governor".format(
multiprocessing.cpu_count() - 1)) as f:
mode = f.read()
print("Performance: {0}".format(mode.rstrip()))
if notify:
show_notification("Performance mode", "{0}".format(mode))
def main():
if len(sys.argv) != 2 and len(sys.argv) != 3 or sys.argv[1] == '-h':
help()
return
queue = multiprocessing.Queue()
value = multiprocessing.Value('i', 0)
processes = []
count = multiprocessing.cpu_count()
szc_dir = sys.argv[2] if len(sys.argv) == 3 else None
for i in range(count):
converter = Converter(sys.argv[1], szc_dir, i, value, queue)
p = multiprocessing.Process(target=converter.convert)
processes.append(p)
p.start()
sys.stdout.write("\033[2J")
while count:
m = queue.get()
if m[1] is None:
count -= 1
else:
sys.stdout.write("\033[%d;0H" % (int(m[0])+1))
print "%60.60s %s/%s f:%s %s/%s " % m[1:]
for p in processes:
p.join()
sys.stdout.write("\033[%d;0H" % (multiprocessing.cpu_count() + 1,))
print("DONE!!! ")
def take_screenshots_async(source_folder, config_path, verbose=False): sizes = load_ss_config(config_path) slides = parse_slide_folders(source_folder) dests = list(map(lambda pair: pair[0], slides[0])) urls = list(map(lambda pair: os.path.join(pair[0], pair[1]), slides[0])) shots = list(gen_configs(urls, dests, sizes, local_slide_name)) q = mp.JoinableQueue() procs = [] for i in range(mp.cpu_count()*2): p = mp.Process(target=ss_q, args=(q,verbose)) procs.append(p) p.start() for item in shots: q.put(tuple(item)) q.join() for i in range(mp.cpu_count()*2): q.put(None) for proc in procs: proc.join()
def parallel(df, func):
if len(df) > 0:
p = Pool(cpu_count())
df = p.map(func, np.array_split(df, cpu_count()))
df = pd.concat(df, axis=0, ignore_index=True).reset_index(drop=True)
p.close(); p.join()
return df
def concurrent_test_jobs(platform):
if platform == "windows":
return str(multiprocessing.cpu_count() // 4)
elif platform == "macos":
return str(multiprocessing.cpu_count() // 2)
else:
return str(multiprocessing.cpu_count())
def main():
total_work = multiprocessing.cpu_count()
burnin = 30000
significance_samples = 100000
per_process_samples = significance_samples / multiprocessing.cpu_count()
alpha_count_slow = 0.001
alpha_count_fast = find_optimal_decay(alpha_count_slow)
alpha_mu_slow = 0.01
alpha_mu_fast = 0.01
buckets_slow = 50
buckets_fast = 50
pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
runs = pool.map(
ergodic_chain,
[[burnin, per_process_samples,
alpha_count_slow, alpha_count_fast,
alpha_mu_slow, alpha_mu_fast,
buckets_slow, buckets_fast] for _ in range(total_work)])
aggregator = [[] for _ in range(len(FUNC_LIST))]
for run in runs:
for i, data_list in enumerate(run):
aggregator[i] += data_list
colors = ['red', 'green', 'blue', 'purple']
for label, data in zip(FUNC_LABELS, aggregator):
#data.sort()
_, _, patches = pylab.hist(
data, 250, label=label,
normed=True, histtype='stepfilled')
pylab.setp(patches, 'alpha', 0.4)
pylab.legend()
pylab.show()
def ProcessStuff(spp_list,epoch_list,model_list):
print 'cpu_count() = %d\n' % multiprocessing.cpu_count()
NUMBER_OF_PROCESSES = multiprocessing.cpu_count()
task_queue = Queue()
done_queue = Queue()
for spp in spp_list:
for model in model_list:
for epoch in epoch_list:
TASKS = [(CallMaxEnt,(spp.rstrip("\r\n"),epoch.rstrip("\r\n"),model.rstrip("\r\n"),str(i))) for i in range(10)]
#print "Number of projections to be made = %d\n" % len(TASKS)
#print TASKS
print " "+spp
# Submit tasks
for task in TASKS:
#print task,"\n"
task_queue.put(task)
# Start worker processes
for i in range(NUMBER_OF_PROCESSES):
Process(target=worker, args=(task_queue, done_queue)).start()
# Get and print results
print 'Unordered results for '+spp.rstrip("\r\n")+':'
for i in range(len(TASKS)):
print '\t', done_queue.get()
# Tell child processes to stop
for i in range(NUMBER_OF_PROCESSES):
task_queue.put('STOP')
def retrieve_clusters(ne_lat, ne_lng, sw_lat, sw_lng, start_date, end_date, fatal, severe, light, inaccurate, zoom):
marker_boxes = divide_to_boxes(ne_lat, ne_lng, sw_lat, sw_lng)
result_futures = []
logging.info("number of cores: " + str(multiprocessing.cpu_count()))
with concurrent.futures.ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) as executor:
for marker_box in marker_boxes:
markers_in_box = Marker.bounding_box_query(
marker_box[0],
marker_box[1],
marker_box[2],
marker_box[3],
start_date,
end_date,
fatal,
severe,
light,
inaccurate,
).all()
result_futures.append(executor.submit(calculate_clusters, markers_in_box, zoom))
completed_futures = concurrent.futures.wait(result_futures)
result = []
for future in completed_futures.done:
result.extend(future.result())
return result
def check_num_cpus(n_cpus, table_size, min_table_size):
messages = []
if multiprocessing.current_process().daemon and n_cpus != 1:
messages.append("WARNING: you choose n_cpus = %d but integrate already runs inside a "
"daemon process which is not allowed. therefore set n_cpus = 1" % n_cpus)
n_cpus = 1
if n_cpus < 0:
n_cpus = multiprocessing.cpu_count() + n_cpus
if n_cpus <= 0:
messages.append("WARNING: you requested to use %d cores, "
"we use single core instead !" % n_cpus)
n_cpus = 1
if n_cpus > 1 and table_size < min_table_size:
messages.append("INFO: as the table has les thann %d rows, we switch to one cpu mode"
% min_table_size)
n_cpus = 1
elif n_cpus > multiprocessing.cpu_count():
messages.append("WARNING: more processes demanded than available cpu cores, this might be "
"inefficient")
return messages, n_cpus
def generate_targets(samples, class_counter):
"""
Generates array of segmented images.
samples: list
list of Sample objects
class_counter: ClassCounter object
object used for generating class markings (class ordinal numbers)
returns: np.array
array of class ordinal numbers
"""
y_shape = (len(samples), requested_shape[0], requested_shape[1])
y = np.zeros(y_shape, dtype='int8')
logger.info("Segmented images new shape %s", y.shape)
pool = mp.Pool(mp.cpu_count())
logger.info("Cpu count %d", mp.cpu_count())
result_func = lambda result: save_result_segm(y, result)
for i, sample in enumerate(samples):
# result_func(mark_image(i, sample.marked_image,
# class_counter, requested_shape))
pool.apply_async(mark_image,
args=(i, sample.marked_image,
class_counter, requested_shape,),
callback=result_func)
pool.close()
pool.join()
return y
# This:
# example "testName" : [["--param1", "--param2"] , ["--param3"]]
# will run the test 3 times:
# testName
# testName --param1 --param2
# testname --param3
"wallet_txn_doublespend.py": [["--mineblock"]],
"wallet_txn_clone.py": [["--mineblock"]],
"wallet_multiwallet.py": [["--usecli"]],
}
# Used to limit the number of tests, when list of tests is not provided on command line
# When --extended is specified, we run all tests, otherwise
# we only run a test if its execution time in seconds does not exceed EXTENDED_CUTOFF
DEFAULT_EXTENDED_CUTOFF = 40
DEFAULT_JOBS = (multiprocessing.cpu_count() // 3) + 1
class TestCase():
"""
Data structure to hold and run information necessary to launch a test case.
"""
def __init__(self, test_num, test_case, tests_dir, tmpdir, flags=None):
self.tests_dir = tests_dir
self.tmpdir = tmpdir
self.test_case = test_case
self.test_num = test_num
self.flags = flags
def run(self, portseed_offset):
t = self.test_case
def generate_file_pairs(
location: Path,
ignore_todo: bool,
strict: bool,
unsafe: bool,
json_output: bool,
save_test_output_tar: bool = True,
) -> None:
filenames = list(location.rglob("*"))
config_filenames = [
filename
for filename in filenames
if filename.suffix in YML_EXTENSIONS
and not filename.name.startswith(".")
and not filename.parent.name.startswith(".")
]
config_test_filenames = {
config_filename: [
inner_filename
for inner_filename in filenames
if inner_filename.with_suffix("") == config_filename.with_suffix("")
and inner_filename.is_file()
and inner_filename.suffix not in YML_EXTENSIONS
]
for config_filename in config_filenames
}
config_with_tests, config_without_tests = partition(
lambda c: c[1], config_test_filenames.items()
)
config_missing_tests_output = [str(c[0]) for c in config_without_tests]
invoke_semgrep_fn = functools.partial(
invoke_semgrep_multi,
no_git_ignore=True,
no_rewrite_rule_ids=True,
strict=strict,
dangerously_allow_arbitrary_code_execution_from_rules=unsafe,
testing=True,
)
with multiprocessing.Pool(multiprocessing.cpu_count()) as pool:
results = pool.starmap(invoke_semgrep_fn, config_with_tests)
config_with_errors, config_without_errors = partition(lambda r: r[1], results)
config_with_errors_output = [
{"filename": str(filename), "error": str(error), "output": output}
for filename, error, output in config_with_errors
]
tested = {
filename: score_output_json(
output, config_test_filenames[filename], ignore_todo
)
for filename, _, output in config_without_errors
}
results_output: Mapping[str, Mapping[str, Any]] = {
str(filename): {
"todo": todo,
"checks": {
check_id: {
"tp": tp,
"tn": tn,
"fp": fp,
"fn": fn,
"passed": (fp == 0) and (fn == 0),
"matches": matches[check_id],
}
for check_id, (tp, tn, fp, fn) in output.items()
},
}
for filename, (output, matches, todo) in tested.items()
}
output = {
"config_missing_tests": config_missing_tests_output,
"config_with_errors": config_with_errors_output,
"results": results_output,
}
strict_error = bool(config_with_errors_output) and strict
any_failures = any(
not check_results["passed"]
for file_results in results_output.values()
for check_results in file_results["checks"].values()
)
exit_code = int(strict_error or any_failures)
if json_output:
print(json.dumps(output, indent=4, separators=(",", ": ")))
sys.exit(exit_code)
# save the results to json file and tar the file to upload as github artifact.
if save_test_output_tar:
list_to_output = []
with open(SAVE_TEST_OUTPUT_JSON, "w") as f:
for tup in results:
true_result = tup[2]
list_to_output.append(true_result)
f.write(json.dumps(list_to_output, indent=4, separators=(",", ":")))
with tarfile.open(SAVE_TEST_OUTPUT_TAR, "w:gz") as tar:
tar.add(SAVE_TEST_OUTPUT_JSON)
if config_missing_tests_output:
print("The following config files are missing tests:")
print("\t" + "\n\t".join(config_missing_tests_output))
if config_with_errors_output:
print("The following config files produced errors:")
print(
"\t"
+ "\n\t".join(
f"{c['filename']}: {c['error']}" for c in config_with_errors_output
)
)
# Place failed tests at the bottom for higher visibility
passed_results_first = collections.OrderedDict(
sorted(
results_output.items(),
key=lambda t: any(not c["passed"] for c in t[1]["checks"].values()),
)
)
print(f"{len(tested)} yaml files tested")
print("check id scoring:")
print("=" * 80)
totals: Dict[str, Any] = collections.defaultdict(int)
for filename, rr in passed_results_first.items():
print(f"(TODO: {rr['todo']}) {filename}")
for check_id, check_results in rr["checks"].items():
print(generate_check_output_line(check_id, check_results))
if not check_results["passed"]:
print(generate_matches_line(check_results))
for confusion in ["tp", "tn", "fp", "fn"]:
totals[confusion] += check_results[confusion]
print("=" * 80)
print(f"final confusion matrix: {generate_confusion_string(totals)}")
print("=" * 80)
sys.exit(exit_code)
## If the running OS is linux; print loadavg.
if (platform == 'Linux'):
print("Linux Load Avarages:" + str(os.getloadavg()))
# Assigment 3
print('\n')
print('Assigment 3')
## Take and print “5 min loadavg” value and cpu core count. If the loadavg value is near
## (or close) to the cpu core count (hint: nproc - 5min loadavg < 1) then exit script.
load_avg1, load_avg5, load_avg15 = os.getloadavg()
# take CPU count
cpu_count = multiprocessing.cpu_count()
# If the loadavg values is near to CPU core count then exit script.
if ((cpu_count - load_avg5) < 1):
exit()
#Assigment 4
print('\n')
print('Assigment 4')
URLs = [
'https://api.github.com', 'http://bilgisayar.mu.edu.tr/',
'https://www.python.org/', 'http://akrepnalan.com/ceng2034',
'https://github.com/caesarsalad/wow'
]
def get_render_cmd(self):
self.renderSettings = {}
self.mappings = {}
render_cmd = ''
if self.G_RENDER_OS == '0':
if float(self.CG_VERSION) < 2016:
version_name = "%s-x64" % (self.CG_VERSION)
else:
version_name = self.CG_VERSION
self.renderSettings["render.exe"] = "/usr/autodesk/" \
"maya%s/bin/Render" % (version_name)
self.renderSettings["mayabatch.exe"] = "/usr/autodesk/" \
"maya%s/bin/maya -batch" % (version_name)
self.renderSettings["render.exe"] = "C:/Program Files/Autodesk/" \
"maya%s/bin/render.exe" % (self.CG_VERSION)
self.renderSettings["output"] = os.path.normpath(
self.G_WORK_RENDER_TASK_OUTPUT).replace("\\", "/")
# 一机多帧
self.renderSettings[
"g_one_machine_multiframe"] = self.g_one_machine_multiframe
if self.g_one_machine_multiframe is True:
self.renderSettings["output"] = os.path.join(
os.path.normpath(self.G_WORK_RENDER_TASK_OUTPUT),
"temp_out").replace("\\", "/")
self.renderSettings["output_frame"] = os.path.normpath(
self.G_WORK_RENDER_TASK_OUTPUT).replace("\\", "/")
if not os.path.exists(self.renderSettings["output"]):
os.makedirs(self.renderSettings["output"])
self.renderSettings["tile_region"] = ""
self.renderSettings["tiles"] = int(self.G_CG_TILE_COUNT)
self.renderSettings["tile_index"] = int(self.G_CG_TILE)
# -----------render tiles------------
if self.renderSettings["tiles"] > 1:
tile_region = self.get_region(
int(self.G_CG_TILE_COUNT), int(self.G_CG_TILE),
int(self.G_TASK_JSON_DICT['scene_info_render'][
self.G_CG_LAYER_NAME]['common']['width']),
int(self.G_TASK_JSON_DICT['scene_info_render'][
self.G_CG_LAYER_NAME]['common']['height']))
self.renderSettings["tile_region"] = " ".join(
[str(i) for i in tile_region])
self.renderSettings["output"] = "%s/%s/%s/" % \
(os.path.normpath(self.G_WORK_RENDER_TASK_OUTPUT).replace("\\", "/"),self.G_CG_START_FRAME,
self.renderSettings["tile_index"])
self.renderSettings["output"] = os.path.normpath(
self.renderSettings["output"])
if not os.path.exists(self.renderSettings["output"]):
os.makedirs(self.renderSettings["output"])
if self.G_INPUT_PROJECT_PATH:
if os.path.exists(self.G_INPUT_PROJECT_PATH):
os.chdir(self.G_INPUT_PROJECT_PATH)
self.renderSettings["maya_file"] = os.path.normpath(
self.G_INPUT_CG_FILE)
self.renderSettings["start"] = self.G_CG_START_FRAME
self.renderSettings["end"] = self.G_CG_END_FRAME
self.renderSettings["by"] = self.G_CG_BY_FRAME
self.renderSettings["renderableCamera"] = self.G_CG_OPTION
self.renderSettings["renderableLayer"] = self.G_CG_LAYER_NAME
self.renderSettings["projectPath"] = os.path.normpath(
self.G_INPUT_PROJECT_PATH)
self.renderSettings["renderType"] = "render.exe"
if self.ENABLE_LAYERED == "1":
self.renderSettings["width"] = int(
self.G_TASK_JSON_DICT['scene_info_render'][
self.G_CG_LAYER_NAME]['common']['width'])
self.renderSettings["height"] = int(
self.G_TASK_JSON_DICT['scene_info_render'][
self.G_CG_LAYER_NAME]['common']['height'])
else:
self.renderSettings["width"] = int(
self.G_TASK_JSON_DICT['scene_info_render']
['defaultRenderLayer']['common']['width'])
self.renderSettings["height"] = int(
self.G_TASK_JSON_DICT['scene_info_render']
['defaultRenderLayer']['common']['height'])
#"-----------------------------cmd--------------------------------"
cmd = "\"%(render.exe)s\" -s %(start)s -e %(end)s -b %(by)s " \
"-proj \"%(projectPath)s\" -rd \"%(output)s\"" \
% self.renderSettings
if self.G_CG_OPTION:
cmd += " -cam \"%(renderableCamera)s\"" % self.renderSettings
if self.G_CG_LAYER_NAME:
cmd += " -rl \"%(renderableLayer)s\"" % self.renderSettings
pre_render_dict = {}
pre_render_dict["enable_layered"] = self.ENABLE_LAYERED
pre_render_dict["projectPath"] = os.path.normpath(
self.G_INPUT_PROJECT_PATH)
pre_render_dict["mapping"] = self.mappings
pre_render_dict["renderableCamera"] = self.G_CG_OPTION
pre_render_dict["renderableLayer"] = self.G_CG_LAYER_NAME
pre_render_dict["task_json"] = self.G_TASK_JSON
pre_render_dict["start"] = self.G_CG_START_FRAME
pre_render_dict["c_prerender"] = self.G_RN_MAYA_CUSTOME_PRERENDER
pre_render_dict["user_id"] = self.G_USER_ID
pre_render_dict["task_id"] = self.G_TASK_ID
pre_render_dict["plugins"] = self.G_CG_CONFIG_DICT["plugins"]
# self.MAYA_BASE_RENDER_CMD = cmd
# cmd += " -preRender \"python \\\"pre_render_dict=%s;execfile(\\\\\\\"%s\\\\\\\")\\\"\"" % (pre_render_dict,self.G_RN_MAYA_PRERENDER)
#---------------render cmd-------------------------
# ----------render tile------------
if self.ENABLE_LAYERED == "1":
self.renderer = self.G_TASK_JSON_DICT['scene_info_render'][
self.G_CG_LAYER_NAME]['common']['renderer']
if self.renderSettings["tile_region"]:
if self.renderer in ["mentalRay", "arnold", "vray"]:
if self.renderer == "mentalRay" and float(
self.CG_VERSION) < 2017:
cmd += " -r mr -reg %(tile_region)s" % self.renderSettings
elif self.renderer == "mentalRay" and float(
self.CG_VERSION
) > 2016.5 and "mentalray" in self.CG_PLUGINS_DICT:
cmd += " -r mr -reg %(tile_region)s" % self.renderSettings
elif self.renderer == "arnold" and "mtoa" in self.CG_PLUGINS_DICT:
cmd += " -r arnold -reg %(tile_region)s" % self.renderSettings
elif self.renderer == "vray" and "vrayformaya" in self.CG_PLUGINS_DICT:
cmd += " -r vray -reg %(tile_region)s" % self.renderSettings
else:
print("please confirm the renderer is correct!")
print(
"current render layer \'s render is %s ,not in [mentalRay,arnold,vray]"
% (self.renderer))
sys.exit(555)
if self.renderer in ["mayaSoftware"]:
cmd += " -r sw -reg %(tile_region)s" % self.renderSettings
else:
if self.renderer == "renderman" and "RenderMan_for_Maya" in self.CG_PLUGINS_DICT:
cmd += " -r rman"
if self.renderer == "vray" and "vrayformaya" in self.CG_PLUGINS_DICT:
cmd += " -r vray"
if self.renderer == "redshift" and "redshift_GPU" in self.CG_PLUGINS_DICT:
cmd += " -r redshift -logLevel 1"
gpu_n = "0,1"
cmd += " -gpu {%s}" % (gpu_n)
else:
pass
else:
scene_info_render_dict = self.G_TASK_JSON_DICT['scene_info_render']
renderer_list = CLASS_MAYA_UTIL.dict_get(scene_info_render_dict,
"renderer")
self.G_DEBUG_LOG.info(renderer_list)
if "redshift_GPU" in self.CG_PLUGINS_DICT and "redshift" in renderer_list:
cmd += " -r redshift -logLevel 1"
gpu_n = "0,1"
cmd += " -gpu {%s}" % (gpu_n)
if "RenderMan_for_Maya" in self.CG_PLUGINS_DICT and "renderman" in renderer_list:
cmd += " -r rman"
max_threads_number = int(multiprocessing.cpu_count())
if " -r " not in cmd and float(self.CG_VERSION) < 2017:
# cmd += " -mr:art -mr:aml"
cmd += " -mr:rt %s -mr:aml" % max_threads_number
# self.RENDERCMD = cmd
options = {}
options["output"] = self.renderSettings["output"]
options["output_frame"] = self.renderSettings["output_frame"]
options["g_one_machine_multiframe"] = self.g_one_machine_multiframe
#-------------get custom render cmd-------------
# -------------add post render cmd-------------
if self.g_one_machine_multiframe is True:
cmd += " -postFrame \"python \\\"options=%s;execfile(\\\\\\\"%s\\\\\\\")\\\";\"" % (
options, self.G_RN_MAYA_POSTRENDER)
if "-r rman" in cmd:
cmd += " -setAttr Format:resolution \"%(width)s %(height)s\" \"%(maya_file)s\"" % self.renderSettings
else:
cmd += " -x %(width)s -y %(height)s \"%(maya_file)s\"" % self.renderSettings
print("render cmd info:")
sys.stdout.flush()
return cmd
def __init__(self, params):
self.param_br = 0.0 # Daily birth rate
self.param_dr = 0.0 # Daily mortality rate except infected people
self.param_vr = 0.0 # Daily vaccination rate (Ratio of susceptible
# population getting vaccinated)
self.param_vir = 0.0 # Ratio of the immunized after vaccination
self.param_mir = 0.0 # Maternal immunization rate
self.param_beta_exp = params[0]/100.0 # Susceptible to exposed transition constant
self.param_qr = params[1]/100.0 # Daily quarantine rate (Ratio of Exposed getting Quarantined)
self.param_beta_inf = 0.0 # Susceptible to infected transition constant
self.param_sir = params[2]/100.0 # Daily isolation rate (Ratio of Infected getting Isolated)
self.param_eps_exp = params[3]/100.0 # Disease transmission rate of exposed compared to the infected
self.param_eps_qua = params[4]/100.0 # Disease transmission rate of quarantined compared to the infected
self.param_eps_sev = params[5]/100.0 # Disease transmission rate of Severe Infected compared to Infected
self.param_hosp_capacity = params[6] # Maximum amount patients that hospital can accommodate
self.param_gamma_mor = 0.0 # Infected to Dead transition probability
self.param_gamma_mor1 = params[7]/100.0 # Severe Infected to Dead transition probability
self.param_gamma_mor2 = params[8]/100.0 # # Severe Infected to Dead transition probability (Hospital Cap. Exceeded)
self.param_gamma_im = params[9]/100.0 # Infected to Recovery Immunized transition probability
self.param_dt = 1/12 # Sampling time in days (1/24 corresponds to one hour)
self.param_sim_len = params[10] # Length of simulation in days
self.param_num_states = 0 # Number of states
self.param_num_sim = int(self.param_sim_len / self.param_dt) + 1 # Number of simulation
self.param_t_exp = params[11] # Incubation period (The period from the start of
# incubation to the end of the incubation state)
self.param_t_inf = params[12] # Infection period (The period from the start of
# infection to the end of the infection state)
self.param_t_vac = 3 # Vaccination immunization period (The time to
# vaccinatization immunization after being vaccinated)
self.param_n_exp = int(self.param_t_exp / self.param_dt)
self.param_n_inf = int(self.param_t_inf / self.param_dt)
self.param_n_vac = int(self.param_t_vac / self.param_dt)
self.param_save_res = 1
self.param_disp_progress = 1
self.param_disp_interval = 100
self.param_vis_on = 1 # Visualize results after simulation
np.random.seed(1)
self.param_rand_seed = np.random.randint(low = 1, high = 100, size = 625)
# Define the initial values for the states
self.init_susceptible = params[13]
self.init_exposed = params[14]
self.init_quarantined = 0.0
self.init_infected = 0.0
self.init_isolated = 0.0
self.init_vaccination_imm = 0.0
self.init_maternally_imm = 0.0
self.init_recovery_imm = 0.0
self.init_severe_infected = 0
# Define states
self.states_x = [0, self.init_susceptible]
self.states_dx = []
self.states_name = ['Birth', 'Susceptible']
self.states_type = ['Birth', 'Susceptible']
# Define transitions
self.source = ['Birth', 'Birth']
self.dest = ['Susceptible', 'Maternally_Immunized' ]
self.source_ind = []
self.dest_ind = []
self.num_cores = multiprocessing.cpu_count()
from spams import omp, ompMask, lasso, lassoMask, somp, l1L2BCD, cd
import spams
import time
from ksvd import KSVD_Encode
from skimage import io
from sklearn.feature_extraction import image
from sklearn.linear_model import lasso_path, orthogonal_mp
from scipy.fftpack import dct
import matplotlib.pyplot as plt
from joblib import Parallel, delayed
import multiprocessing
from threading import Thread
num_cores = multiprocessing.cpu_count()
par = Parallel(n_jobs=num_cores)
def clip(img):
img = np.minimum(np.ones(img.shape), img)
img = np.maximum(np.zeros(img.shape), img)
return img
def recover_same_kron(p, kronprod):
y = p.T
print(y.shape, y.dtype)
phikron = kronprod.T
print(phikron.shape)
sx = spams.omp(np.asfortranarray(y), np.asfortranarray(phikron), eps=0.001, L=6)
# sx = orthogonal_mp(phikron, y, tol=0.01)
def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
clean_dir=False):
"""The main control code for buildman
Args:
options: Command line options object
args: Command line arguments (list of strings)
toolchains: Toolchains to use - this should be a Toolchains()
object. If None, then it will be created and scanned
make_func: Make function to use for the builder. This is called
to execute 'make'. If this is None, the normal function
will be used, which calls the 'make' tool with suitable
arguments. This setting is useful for tests.
board: Boards() object to use, containing a list of available
boards. If this is None it will be created and scanned.
"""
global builder
if options.full_help:
pager = os.getenv('PAGER')
if not pager:
pager = 'more'
fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
'README')
command.Run(pager, fname)
return 0
gitutil.Setup()
col = terminal.Color()
options.git_dir = os.path.join(options.git, '.git')
no_toolchains = toolchains is None
if no_toolchains:
toolchains = toolchain.Toolchains(options.override_toolchain)
if options.fetch_arch:
if options.fetch_arch == 'list':
sorted_list = toolchains.ListArchs()
print(col.Color(col.BLUE, 'Available architectures: %s\n' %
' '.join(sorted_list)))
return 0
else:
fetch_arch = options.fetch_arch
if fetch_arch == 'all':
fetch_arch = ','.join(toolchains.ListArchs())
print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
fetch_arch))
for arch in fetch_arch.split(','):
print()
ret = toolchains.FetchAndInstall(arch)
if ret:
return ret
return 0
if no_toolchains:
toolchains.GetSettings()
toolchains.Scan(options.list_tool_chains and options.verbose)
if options.list_tool_chains:
toolchains.List()
print()
return 0
# Work out how many commits to build. We want to build everything on the
# branch. We also build the upstream commit as a control so we can see
# problems introduced by the first commit on the branch.
count = options.count
has_range = options.branch and '..' in options.branch
if count == -1:
if not options.branch:
count = 1
else:
if has_range:
count, msg = gitutil.CountCommitsInRange(options.git_dir,
options.branch)
else:
count, msg = gitutil.CountCommitsInBranch(options.git_dir,
options.branch)
if count is None:
sys.exit(col.Color(col.RED, msg))
elif count == 0:
sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
options.branch))
if msg:
print(col.Color(col.YELLOW, msg))
count += 1 # Build upstream commit also
if not count:
str = ("No commits found to process in branch '%s': "
"set branch's upstream or use -c flag" % options.branch)
sys.exit(col.Color(col.RED, str))
# Work out what subset of the boards we are building
if not boards:
board_file = os.path.join(options.output_dir, 'boards.cfg')
genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
status = subprocess.call([genboardscfg, '-o', board_file])
if status != 0:
sys.exit("Failed to generate boards.cfg")
boards = board.Boards()
boards.ReadBoards(board_file)
exclude = []
if options.exclude:
for arg in options.exclude:
exclude += arg.split(',')
if options.boards:
requested_boards = []
for b in options.boards:
requested_boards += b.split(',')
else:
requested_boards = None
why_selected, board_warnings = boards.SelectBoards(args, exclude,
requested_boards)
selected = boards.GetSelected()
if not len(selected):
sys.exit(col.Color(col.RED, 'No matching boards found'))
# Read the metadata from the commits. First look at the upstream commit,
# then the ones in the branch. We would like to do something like
# upstream/master~..branch but that isn't possible if upstream/master is
# a merge commit (it will list all the commits that form part of the
# merge)
# Conflicting tags are not a problem for buildman, since it does not use
# them. For example, Series-version is not useful for buildman. On the
# other hand conflicting tags will cause an error. So allow later tags
# to overwrite earlier ones by setting allow_overwrite=True
if options.branch:
if count == -1:
if has_range:
range_expr = options.branch
else:
range_expr = gitutil.GetRangeInBranch(options.git_dir,
options.branch)
upstream_commit = gitutil.GetUpstream(options.git_dir,
options.branch)
series = patchstream.GetMetaDataForList(upstream_commit,
options.git_dir, 1, series=None, allow_overwrite=True)
series = patchstream.GetMetaDataForList(range_expr,
options.git_dir, None, series, allow_overwrite=True)
else:
# Honour the count
series = patchstream.GetMetaDataForList(options.branch,
options.git_dir, count, series=None, allow_overwrite=True)
else:
series = None
if not options.dry_run:
options.verbose = True
if not options.summary:
options.show_errors = True
# By default we have one thread per CPU. But if there are not enough jobs
# we can have fewer threads and use a high '-j' value for make.
if not options.threads:
options.threads = min(multiprocessing.cpu_count(), len(selected))
if not options.jobs:
options.jobs = max(1, (multiprocessing.cpu_count() +
len(selected) - 1) // len(selected))
if not options.step:
options.step = len(series.commits) - 1
gnu_make = command.Output(os.path.join(options.git,
'scripts/show-gnu-make'), raise_on_error=False).rstrip()
if not gnu_make:
sys.exit('GNU Make not found')
# Create a new builder with the selected options.
output_dir = options.output_dir
if options.branch:
dirname = options.branch.replace('/', '_')
# As a special case allow the board directory to be placed in the
# output directory itself rather than any subdirectory.
if not options.no_subdirs:
output_dir = os.path.join(options.output_dir, dirname)
if clean_dir and os.path.exists(output_dir):
shutil.rmtree(output_dir)
CheckOutputDir(output_dir)
builder = Builder(toolchains, output_dir, options.git_dir,
options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
show_unknown=options.show_unknown, step=options.step,
no_subdirs=options.no_subdirs, full_path=options.full_path,
verbose_build=options.verbose_build,
incremental=options.incremental,
per_board_out_dir=options.per_board_out_dir,
config_only=options.config_only,
squash_config_y=not options.preserve_config_y,
warnings_as_errors=options.warnings_as_errors)
builder.force_config_on_failure = not options.quick
if make_func:
builder.do_make = make_func
# For a dry run, just show our actions as a sanity check
if options.dry_run:
ShowActions(series, why_selected, selected, builder, options,
board_warnings)
else:
builder.force_build = options.force_build
builder.force_build_failures = options.force_build_failures
builder.force_reconfig = options.force_reconfig
builder.in_tree = options.in_tree
# Work out which boards to build
board_selected = boards.GetSelectedDict()
if series:
commits = series.commits
# Number the commits for test purposes
for commit in range(len(commits)):
commits[commit].sequence = commit
else:
commits = None
Print(GetActionSummary(options.summary, commits, board_selected,
options))
# We can't show function sizes without board details at present
if options.show_bloat:
options.show_detail = True
builder.SetDisplayOptions(options.show_errors, options.show_sizes,
options.show_detail, options.show_bloat,
options.list_error_boards,
options.show_config,
options.show_environment)
if options.summary:
builder.ShowSummary(commits, board_selected)
else:
fail, warned = builder.BuildBoards(commits, board_selected,
options.keep_outputs, options.verbose)
if fail:
return 128
elif warned:
return 129
return 0
del df_m
#df_predicted = df_predicted.append(df_temp, ignore_index=True)
print(df_temp.shape)
print(" PRN : " + str(prn) + " acc : " + str(avg_acc / 30))
return (df_temp, avg_acc, prn)
if __name__ == '__main__':
total_err = []
prn_list = []
a = datetime.datetime.now().replace(microsecond=0)
input_prn = df_final['prn'].unique()
pool = mp.Pool(processes=mp.cpu_count())
for df_pred, err, prn in pool.map(predict_each_satellite, input_prn):
df_predicted = df_predicted.append(df_pred, ignore_index=True)
total_err.append(err)
prn_list.append(prn)
pool.close()
pool.join()
b = datetime.datetime.now().replace(microsecond=0)
print(b - a)
df_predicted.shape
df_predicted
df_predicted.reset_index()
import multiprocessing
from multiprocessing import Pool
import os
def wrapper(f):
os.system('python runner_personal.py -t 12 ' + mydir + f)
if __name__ == '__main__':
os.chdir('/stor/work/Ochman/brian/ConSpeciFix/database/')
mydir = '/stor/work/Ochman/brian/dengueMix/'
print multiprocessing.cpu_count()
p = Pool(5)
args = os.listdir(mydir)
p.map(wrapper, args)
def machine_cpu_count():
"""
:return: Retrieves the machine's CPU count
:rtype: int
"""
return multiprocessing.cpu_count()
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
"""
Demonstration of embarrassingly parallel loops so I can understand / remember
how to apply this to the rucio script
"""
import numpy as np
import multiprocessing
from multiprocessing import Pool
import time
MAXTHREADS=multiprocessing.cpu_count()
print MAXTHREADS
USETHREADS=5
def f(x):
return x*np.random.randn(100000)
if __name__ == '__main__':
s = time.time()
p = Pool(processes=USETHREADS)
result = p.map(f, [1, 2, 3])
time_taken = time.time() - s
print("Time taken in pool: {}".format(time_taken))
s = time.time()
def main():
# Start logging
logger = logging.getLogger("lg")
logger.setLevel(logging.INFO)
formatter = logging.Formatter(fmt='%(asctime)s - %(module)s - %(levelname)s - %(message)s')
handler = logging.FileHandler('%s.log' % os.path.splitext(os.path.basename(__file__))[0])
handler.setLevel(logging.INFO)
handler.setFormatter(formatter)
logger.addHandler(handler)
streamhandler = logging.StreamHandler()
streamhandler.setLevel(logging.INFO)
streamhandler.setFormatter(formatter)
logger.addHandler(streamhandler)
logger.info('Starting %s' % __file__)
# Parse CMD
parser = argparse.ArgumentParser(prog="nucleosomeDynamics_wf", description="Nucleoseom Dynamics workflow")
parser.add_argument("--config", required=True, type=Mugparams.check_json, metavar="CONFIG_JSON",
help="JSON file containing workflow parameters")
parser.add_argument("--root_dir", required=True, type=Mugparams.readable_dir, metavar="ABS_PATH",
help="Absolute path of the user data directory.")
parser.add_argument("--metadata", required=True, type=Mugparams.check_json, metavar="METADATA_JSON",
help="JSON file containing MuG metadata files")
parser.add_argument("--out_metadata", required=True, type=Mugparams.writeable_file, metavar="RESULTS_JSON",
help="JSON file containing results metadata")
parser.add_argument("-v", "--verbose", required=False, action="store_true",
help="increase output verbosity")
parser.add_argument('--version', action='version', version='%(prog)s 0.1')
args = parser.parse_args()
if args.verbose:
logger.setLevel(logging.DEBUG)
handler.setLevel(logging.DEBUG)
handler.setLevel(logging.DEBUG)
logger.addHandler(handler)
streamhandler.setLevel(logging.DEBUG)
logger.addHandler(streamhandler)
logger.debug("Verbose mode on")
# Parse config
Mugparams.process_arguments(args)
# Print host info
num_cores = multiprocessing.cpu_count()
host = socket.gethostname()
#mem = psutil.virtual_memory()
logger.debug('HOST=%s CPUs=%s MEM=x' %(host,num_cores))
# Run pipeline
outfiles = run_pipeline(args, num_cores)
# Results
prepare_results(args)
class _RobustMNINormalizationInputSpec(BaseInterfaceInputSpec):
# Enable deprecation
package_version = niworkflows_version
# Moving image.
moving_image = File(exists=True,
mandatory=True,
desc='image to apply transformation to')
# Reference image (optional).
reference_image = File(exists=True, desc='override the reference image')
# Moving mask (optional).
moving_mask = File(exists=True, desc='moving image mask')
# Reference mask (optional).
reference_mask = File(exists=True, desc='reference image mask')
# Lesion mask (optional).
lesion_mask = File(exists=True, desc='lesion mask image')
# Number of threads to use for ANTs/ITK processes.
num_threads = traits.Int(cpu_count(),
usedefault=True,
nohash=True,
desc="Number of ITK threads to use")
# ANTs parameter set to use.
flavor = traits.Enum('precise',
'testing',
'fast',
usedefault=True,
desc='registration settings parameter set')
# Template orientation.
orientation = traits.Enum(
'RAS',
'LAS',
mandatory=True,
usedefault=True,
desc='modify template orientation (should match input image)')
# Modality of the reference image.
reference = traits.Enum('T1w',
'T2w',
'boldref',
'PDw',
mandatory=True,
usedefault=True,
desc='set the reference modality for registration')
# T1 or EPI registration?
moving = traits.Enum('T1w',
'boldref',
usedefault=True,
mandatory=True,
desc='registration type')
# Template to use as the default reference image.
template = traits.Str('MNI152NLin2009cAsym',
usedefault=True,
desc='define the template to be used')
# Load other settings from file.
settings = traits.List(File(exists=True),
desc='pass on the list of settings files')
# Resolution of the default template.
template_spec = traits.DictStrAny(desc='template specifications')
template_resolution = traits.Enum(1,
2,
desc='(DEPRECATED) template resolution')
# Use explicit masking?
explicit_masking = traits.Bool(True,
usedefault=True,
desc="""\
Set voxels outside the masks to zero thus creating an artificial border
that can drive the registration. Requires reliable and accurate masks.
See https://sourceforge.net/p/advants/discussion/840261/thread/27216e69/#c7ba\
""")
initial_moving_transform = File(exists=True,
desc='transform for initialization')
float = traits.Bool(False,
usedefault=True,
desc='use single precision calculations')
def default_concurrenty():
try:
return int(os.environ.get('THRIFT_CROSSTEST_CONCURRENCY'))
except (TypeError, ValueError):
# Since much time is spent sleeping, use many threads
return int(multiprocessing.cpu_count() * 1.25) + 1
def __init__(self, host=None,
api_key=None, api_key_prefix=None,
username=None, password=None,
discard_unknown_keys=False,
disabled_client_side_validations="",
server_index=None, server_variables=None,
server_operation_index=None, server_operation_variables=None,
):
"""Constructor
"""
self._base_path = "http://localhost:41101" if host is None else host
"""Default Base url
"""
self.server_index = 0 if server_index is None and host is None else server_index
self.server_operation_index = server_operation_index or {}
"""Default server index
"""
self.server_variables = server_variables or {}
self.server_operation_variables = server_operation_variables or {}
"""Default server variables
"""
self.temp_folder_path = None
"""Temp file folder for downloading files
"""
# Authentication Settings
self.api_key = {}
if api_key:
self.api_key = api_key
"""dict to store API key(s)
"""
self.api_key_prefix = {}
if api_key_prefix:
self.api_key_prefix = api_key_prefix
"""dict to store API prefix (e.g. Bearer)
"""
self.refresh_api_key_hook = None
"""function hook to refresh API key if expired
"""
self.username = username
"""Username for HTTP basic authentication
"""
self.password = password
"""Password for HTTP basic authentication
"""
self.discard_unknown_keys = discard_unknown_keys
self.disabled_client_side_validations = disabled_client_side_validations
self.logger = {}
"""Logging Settings
"""
self.logger["package_logger"] = logging.getLogger("regula.facesdk.webclient.gen")
self.logger["urllib3_logger"] = logging.getLogger("urllib3")
self.logger_format = '%(asctime)s %(levelname)s %(message)s'
"""Log format
"""
self.logger_stream_handler = None
"""Log stream handler
"""
self.logger_file_handler = None
"""Log file handler
"""
self.logger_file = None
"""Debug file location
"""
self.debug = False
"""Debug switch
"""
self.verify_ssl = True
"""SSL/TLS verification
Set this to false to skip verifying SSL certificate when calling API
from https server.
"""
self.ssl_ca_cert = None
"""Set this to customize the certificate file to verify the peer.
"""
self.cert_file = None
"""client certificate file
"""
self.key_file = None
"""client key file
"""
self.assert_hostname = None
"""Set this to True/False to enable/disable SSL hostname verification.
"""
self.connection_pool_maxsize = multiprocessing.cpu_count() * 5
"""urllib3 connection pool's maximum number of connections saved
per pool. urllib3 uses 1 connection as default value, but this is
not the best value when you are making a lot of possibly parallel
requests to the same host, which is often the case here.
cpu_count * 5 is used as default value to increase performance.
"""
self.proxy = None
"""Proxy URL
"""
self.proxy_headers = None
"""Proxy headers
"""
self.safe_chars_for_path_param = ''
"""Safe chars for path_param
"""
self.retries = None
"""Adding retries to override urllib3 default value 3
"""
# Enable client side validation
self.client_side_validation = True
'objective': 'binary',
'metric': 'auc',
'learning_rate': 0.01,
'max_depth': 6,
'num_leaves': 63,
'max_bin': 255,
'min_child_weight': 10,
'min_data_in_leaf': 150,
'reg_lambda': 0.5, # L2 regularization term on weights.
'reg_alpha': 0.5, # L1 regularization term on weights.
'colsample_bytree': 0.9,
'subsample': 0.9,
# 'nthread': 32,
'nthread': cpu_count(),
'bagging_freq': 1,
'verbose':-1,
'seed': SEED
}
if ONLY_ME:
use_files = ['train_f']
else:
use_files = ['train_']
drop_ids = pd.read_csv('../data/drop_ids.csv')['SK_ID_CURR']
# =============================================================================
# reset load
def train_rl(
output_dir,
train_batch_size,
eval_batch_size,
env_name='Acrobot-v1',
max_timestep=None,
clip_rewards=False,
rendered_env=False,
resize=False,
resize_dims=(105, 80),
trainer_class=rl_trainers.PPO,
n_epochs=10000,
trajectory_dump_dir=None,
num_actions=None,
light_rl=False,
light_rl_trainer=light_trainers.RLTrainer,
):
"""Train the RL agent.
Args:
output_dir: Output directory.
train_batch_size: Number of parallel environments to use for training.
eval_batch_size: Number of parallel environments to use for evaluation.
env_name: Name of the environment.
max_timestep: Int or None, the maximum number of timesteps in a trajectory.
The environment is wrapped in a TimeLimit wrapper.
clip_rewards: Whether to clip and discretize the rewards.
rendered_env: Whether the environment has visual input. If so, a
RenderedEnvProblem will be used.
resize: whether to do resize or not
resize_dims: Pair (height, width), dimensions to resize the visual
observations to.
trainer_class: RLTrainer class to use.
n_epochs: Number epochs to run the training for.
trajectory_dump_dir: Directory to dump trajectories to.
num_actions: None unless one wants to use the discretization wrapper. Then
num_actions specifies the number of discrete actions.
light_rl: whether to use the light RL setting (experimental).
light_rl_trainer: whichh light RL trainer to use (experimental).
"""
if light_rl:
task = rl_task.RLTask()
env_name = task.env_name
if FLAGS.jax_debug_nans:
config.update('jax_debug_nans', True)
if FLAGS.use_tpu:
config.update('jax_platform_name', 'tpu')
else:
config.update('jax_platform_name', '')
if light_rl:
trainer = light_rl_trainer(task=task, output_dir=output_dir)
trainer.run(n_epochs, n_epochs_is_total_epochs=True)
trainer.close()
return
# TODO(pkozakowski): Find a better way to determine this.
train_env_kwargs = {}
eval_env_kwargs = {}
if 'OnlineTuneEnv' in env_name:
envs_output_dir = FLAGS.envs_output_dir or os.path.join(output_dir, 'envs')
train_env_output_dir = os.path.join(envs_output_dir, 'train')
eval_env_output_dir = os.path.join(envs_output_dir, 'eval')
train_env_kwargs = {'output_dir': train_env_output_dir}
eval_env_kwargs = {'output_dir': eval_env_output_dir}
parallelism = multiprocessing.cpu_count() if FLAGS.parallelize_envs else 1
logging.info('Num discretized actions %s', num_actions)
logging.info('Resize %d', resize)
train_env = env_problem_utils.make_env(
batch_size=train_batch_size,
env_problem_name=env_name,
rendered_env=rendered_env,
resize=resize,
resize_dims=resize_dims,
max_timestep=max_timestep,
clip_rewards=clip_rewards,
parallelism=parallelism,
use_tpu=FLAGS.use_tpu,
num_actions=num_actions,
**train_env_kwargs)
assert train_env
eval_env = env_problem_utils.make_env(
batch_size=eval_batch_size,
env_problem_name=env_name,
rendered_env=rendered_env,
resize=resize,
resize_dims=resize_dims,
max_timestep=max_timestep,
clip_rewards=clip_rewards,
parallelism=parallelism,
use_tpu=FLAGS.use_tpu,
num_actions=num_actions,
**eval_env_kwargs)
assert eval_env
def run_training_loop():
"""Runs the training loop."""
logging.info('Starting the training loop.')
trainer = trainer_class(
output_dir=output_dir,
train_env=train_env,
eval_env=eval_env,
trajectory_dump_dir=trajectory_dump_dir,
async_mode=FLAGS.async_mode,
)
trainer.training_loop(n_epochs=n_epochs)
if FLAGS.jax_debug_nans or FLAGS.disable_jit:
with jax.disable_jit():
run_training_loop()
else:
run_training_loop()
log.info('Removing container: %s' % name)
subproc = subprocess.Popen(['/usr/bin/docker', 'rm', name],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
cmd_stdout, cmd_stderr = subproc.communicate()
if cmd_stdout:
log.debug(cmd_stdout)
if cmd_stderr and \
cmd_stderr != 'Error response from daemon: ' \
'No such container: {}\n'.format(name):
log.debug(cmd_stderr)
process_count = int(
os.environ.get('PROCESS_COUNT', multiprocessing.cpu_count()))
log = get_logger()
log.info('Running docker-puppet')
config_file = os.environ.get('CONFIG',
'/var/lib/docker-puppet/docker-puppet.json')
log.debug('CONFIG: %s' % config_file)
with open(config_file) as f:
json_data = json.load(f)
# To save time we support configuring 'shared' services at the same
# time. For example configuring all of the heat services
# in a single container pass makes sense and will save some time.
# To support this we merge shared settings together here.
#
# We key off of config_volume as this should be the same for a
# given group of services. We are also now specifying the container
from thirdparty.fcrypt.fcrypt import crypt
_multiprocessing = None
try:
import multiprocessing
# problems on FreeBSD (Reference: https://web.archive.org/web/20110710041353/http://www.eggheadcafe.com/microsoft/Python/35880259/multiprocessing-on-freebsd.aspx)
_ = multiprocessing.Queue()
# problems with ctypes (Reference: https://github.com/sqlmapproject/sqlmap/issues/2952)
_ = multiprocessing.Value('i')
except (ImportError, OSError, AttributeError):
pass
else:
try:
if multiprocessing.cpu_count() > 1:
_multiprocessing = multiprocessing
except NotImplementedError:
pass
import base64
import binascii
import gc
import math
import os
import re
import tempfile
import time
import zipfile
from hashlib import md5
import re
import logging
import multiprocessing
import tqdm
from .getter import (
get_films_page,
get_profile_page,
)
from .parser import (
auth_check,
extract_movie_ratings,
get_pages_count,
write_data,
)
PARALLEL_PROC = multiprocessing.cpu_count()
def main():
args = docopt(__doc__)
user = args['<username>']
cookie = args['<cookie>']
assert all([user, cookie]), 'Empty arguments provided'
file_format = (args['--format'] or 'json').lower()
assert file_format in ('all', 'csv', 'json'), 'Supported file formats: all, csv, JSON'
if args['--debug']:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
pool = multiprocessing.Pool(processes=PARALLEL_PROC)
try:
logging.info('Checking args...')
cv2.destroyAllWindows()
print("Video saved by the name Output_Video,mp4:")
if __name__ == '__main__':
# Some variables that store paths:
# TO change the input video or output video path one can change the following variables.
# I was originally planning on doing argpars, but since I am using Pycharm and run scripts from there directly
# I haven't used argparse.
input_path = './../Videos/Input_video.mp4'
output_path = './../Videos/Output_video.mp4'
frames = read_frames(input_path)
frame_seq = np.array_split(frames, multiprocessing.cpu_count())
print("Printing OriginalSizes:")
print("Total: {}, Batch 1: {}, Batch 2: {}, Batch 3: {}, Batch 4: {}, Total Again:{}".format(len(frames),
len(frame_seq[0]),
len(frame_seq[1]),
len(frame_seq[2]),
len(frame_seq[3]),
(len(frame_seq[0]) +
len(frame_seq[1]) +
len(frame_seq[2]) +
len(frame_seq[3])
)
)
)
print()
# see <http://www.gnu.org/licenses/>.
#
from pysix.cphd import CPHDReader, CPHDWriter, Wideband
import sys
import multiprocessing
if __name__ == '__main__':
if len(sys.argv) >= 3:
inputPathname = sys.argv[1]
outputPathname = sys.argv[2]
else:
print("Usage: " + sys.argv[0] + " <Input CPHD> <Output CPHD>")
sys.exit(0)
reader = CPHDReader(inputPathname, multiprocessing.cpu_count())
writer = CPHDWriter(reader.getMetadata())
wideband = reader.getWideband()
writer.writeCPHD(outputPathname, wideband.read(), reader.getVBM(), 0)
roundTrippedReader = CPHDReader(outputPathname,
multiprocessing.cpu_count())
roundTrippedWideband = roundTrippedReader.getWideband()
assert (roundTrippedWideband.read() == wideband.read()).all()
assert (roundTrippedReader.getVBM().toBuffer(0) ==
reader.getVBM().toBuffer(0)).all()
assert (roundTrippedReader.getFileHeader().toString() ==
reader.getFileHeader().toString())
assert roundTrippedReader.getMetadata() == reader.getMetadata()
sys.exit(0)
import time
import secrets
import logging
import random
import logging.handlers
import os
import multiprocessing
CPU_COUNT = multiprocessing.cpu_count()
def log_forver():
last_log_time = 0.0
last_log_length = 1
update_logger = logging.getLogger("update")
while True:
length = random.randint(2, 40)
log_str = f"{secrets.token_urlsafe(length)},{last_log_length} {last_log_time}ms"
if last_log_time > 10:
update_log_str = f"CPU{CPU_COUNT} python write {last_log_length} byte {last_log_time} ms"
if last_log_time > 50:
update_logger.error(update_log_str)
else:
update_logger.info(update_log_str)
before = time.time() * 1000
if last_log_time > 50: # 大于100毫秒
logging.error(log_str)
elif last_log_time > 10:
parser = argparse.ArgumentParser()
# train
parser.add_argument('-lw',
'--load-weights',
type=str,
help='load model weights (and continue training)')
parser.add_argument('-lm',
'--load-model',
type=str,
help='load model (and continue training)')
#num_workers = cpu_count()
parser.add_argument('--num-workers',
type=int,
default=cpu_count(),
help='Workers for multi-thread generators')
#use_metadata = True
parser.add_argument('--no-metadata', action='store_true', help='Use metadata')
#batch_size_cnn = 48
#batch_size_multi = 64
#batch_size_eval = 128
parser.add_argument('-b',
'--batch-size',
type=int,
default=32,
help='Batch size')
#metadata_length = 21
ms_pad = ms + 5 n = int(fs*0.001*ms_pad) fp = open(filename,"rb") x = io.get_samples_complex(fp,n) # resample to 3*10.230 MHz fsr = 3*10230000.0/fs nco.mix(x,-coffset/fs,0) h = scipy.signal.firwin(161,12e6/(fs/2),window='hanning') x = scipy.signal.filtfilt(h,[1],x) xr = np.interp((1/fsr)*np.arange(ms_pad*3*10230),np.arange(len(x)),np.real(x)) xi = np.interp((1/fsr)*np.arange(ms_pad*3*10230),np.arange(len(x)),np.imag(x)) x = xr+(1j)*xi # iterate (in parallel) over PRNs of interest def worker(p): x,prn = p metric,code,doppler = search(x,prn,doppler_search,ms) return 'prn %2d doppler % 7.1f metric % 7.1f code_offset %6.1f' % (prn,doppler,metric,code) import multiprocessing as mp cpus = mp.cpu_count() results = mp.Pool(cpus).map(worker, map(lambda prn: (x,prn),prns)) for r in results: print(r)
xp.Cf1001i.append(vvv)
vvv = (xp.F1001R[j] - xm.F1001R[j]) / (2 * dpp)
xp.Cf1010r.append(vvv)
vvv = (xp.F1010I[j] - xm.F1010I[j]) / (2 * dpp)
xp.Cf1010i.append(vvv)
os.remove(path + "/twiss.dp+." + var)
os.remove(path + "/twiss.dp-." + var)
return var, xp
if __name__ == '__main__':
timeStartGlobal = time.time()
numberofCPUs = multiprocessing.cpu_count()
pool = multiprocessing.Pool(processes=numberofCPUs)
##### optionparser
parser = optparse.OptionParser()
parser.add_option("-a",
"--accel",
help="Which accelerator: LHCB1 LHCB2 SPS RHIC SOLEIL",
default="LHCB1",
dest="accel")
parser.add_option("-p",
"--path",
help="path to save",
default="./",
dest="path")
parser.add_option(
def add_setup_args(parser):
parser.add_argument('--train_url',
type=str,
default='https://github.com/chrischute/squad/data/train-v2.0.json')
parser.add_argument('--dev_url',
type=str,
default='https://github.com/chrischute/squad/data/dev-v2.0.json')
parser.add_argument('--test_url',
type=str,
default='https://github.com/chrischute/squad/data/test-v2.0.json')
parser.add_argument('--glove_url',
type=str,
default='http://nlp.stanford.edu/data/glove.840B.300d.zip')
parser.add_argument('--dev_meta_file',
type=str,
default='./data/dev_meta.json')
parser.add_argument('--test_meta_file',
type=str,
default='./data/test_meta.json')
parser.add_argument('--para_limit',
type=int,
default=400,
help='Max number of words in a paragraph')
parser.add_argument('--ques_limit',
type=int,
default=50,
help='Max number of words to keep from a question')
parser.add_argument('--test_para_limit',
type=int,
default=1000,
help='Max number of words in a paragraph at test time')
parser.add_argument('--test_ques_limit',
type=int,
default=100,
help='Max number of words in a question at test time')
parser.add_argument('--char_dim',
type=int,
default=64,
help='Size of char vectors (char-level embeddings)')
parser.add_argument('--glove_dim',
type=int,
default=300,
help='Size of GloVe word vectors to use')
parser.add_argument('--glove_num_vecs',
type=int,
default=2196017,
help='Number of GloVe vectors')
parser.add_argument('--ans_limit',
type=int,
default=30,
help='Max number of words in a training example answer')
parser.add_argument('--char_limit',
type=int,
default=16,
help='Max number of chars to keep from a word')
parser.add_argument('--wv_file', default='data/glove.840B.300d/glove.840B.300d.txt',
help='path to word vector file.')
parser.add_argument('--wv_dim', type=int, default=300,
help='word vector dimension.')
parser.add_argument('--wv_cased', type=str2bool, nargs='?',
const=True, default=True,
help='treat the words as cased or not.')
parser.add_argument('--sort_all', action='store_true',
help='sort the vocabulary by frequencies of all words. '
'Otherwise consider question words first.')
parser.add_argument('--sample_size', type=int, default=0,
help='size of sample data (for debugging).')
parser.add_argument('--threads', type=int, default=min(multiprocessing.cpu_count(), 16),
help='number of threads for preprocessing.')
parser.add_argument('--batch_size', type=int, default=64,
help='batch size for multiprocess tokenizing and tagging.')
args = parser.parse_args()
return args
def main():
"""Main"""
parser = OptionParser(usage='Usage: %prog [options] eval_dir',
version='Barnum Cluster ' + module_version)
parser.add_option(
'-c',
'--csv',
action='store',
type='str',
default=None,
help='Save CSV of results to given filepath (default: no CSV)')
parser.add_option(
'-p',
'--plot',
action='store',
type='str',
default=None,
help=
'Save plot as a PNG image to the given filepath (default: no plotting)'
)
parser.add_option(
'-w',
'--workers',
action='store',
dest='workers',
type='int',
default=cpu_count(),
help='Number of workers to use (default: number of cores)')
parser.add_option('--max-classes',
action='store',
type='int',
default=256,
help='How many classes to use (default: 256)')
parser.add_option(
'--min-samples',
action='store',
type='int',
default=4,
help='Minimum samples to form a cluster in DBSCAN (default: 4)')
parser.add_option('--eps',
action='store',
type='float',
default=0.03,
help='Epsilon parameter to DBSCAN (default: 0.03)')
options, args = parser.parse_args()
if len(args) != 1 or options.workers < 1:
parser.print_help()
sys.exit(ERROR_INVALID_ARG)
logger.log_start(20)
logger.log_info(module_name, 'Barnum Cluster %s' % module_version)
idirpath = args[0]
if not os.path.isdir(idirpath):
logger.log_error(module_name,
'ERROR: %s is not a directory' % idirpath)
logger.log_stop()
sys.exit(ERROR_INVALID_ARG)
files = [
os.path.join(idirpath, f) for f in os.listdir(idirpath)
if os.path.isfile(os.path.join(idirpath, f))
]
# We only care about clustering malicious traces
mal_files = [fp for fp in files if 'malicious' in os.path.basename(fp)]
num_mal = len(mal_files)
# Calculate clustering metrics
logger.log_info(module_name, "Parsing " + idirpath)
pool = Pool(options.workers)
data = [
sample for sample in pool.map(
parse_file, zip(mal_files, [options.max_classes] * num_mal))
if sample
]
pool.close()
xs = np.array([sample[0] for sample in data])
ns = [sample[1] for sample in data]
# Clustering
logger.log_info(module_name, "Calculating clusters")
db = DBSCAN(eps=options.eps, min_samples=options.min_samples).fit(xs)
core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
core_samples_mask[db.core_sample_indices_] = True
labels = db.labels_
# Number of clusters in labels, ignoring noise if present.
n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
n_noise = list(labels).count(-1)
logger.log_info(module_name, ' Number of points: %d' % len(ns))
logger.log_info(module_name, ' Number of clusters: %d' % n_clusters)
logger.log_info(module_name, 'Number of noise points: %d' % n_noise)
# Saving results as CSV
if not options.csv is None:
logger.log_info(module_name, "Saving CSV to %s" % options.csv)
try:
with open(options.csv, 'w') as csv_file:
csv_file.write("cluster,filename\n")
for label, name in zip(labels, ns):
csv_file.write(','.join([str(label), name]) + "\n")
except Exception as ex:
logger.log_error(module_name, "Failed to save CSV: %s" % str(ex))
# Saving results as plot image
if not options.plot is None:
logger.log_info(module_name, "Generating plot")
theta = radar_factory(options.max_classes, frame='polygon')
fig, axes = plt.subplots(subplot_kw=dict(projection='radar'))
colors = ['b', 'r', 'g', 'm', 'y']
axes.set_varlabels([""]) # no varlabels, they aren't that meaningful
axes.set_rgrids([0.2, 0.4, 0.6, 0.8])
legend_labels = list()
for label_key in set(labels):
if label_key == -1:
continue # noise
legend_labels.append(label_key)
label_color = colors[label_key % len(colors)]
# Calculate per-cluster average
label_mask = (labels == label_key)
label_points = xs[label_mask & core_samples_mask]
label_means = np.mean(label_points, axis=0)
axes.plot(theta, label_means, color=label_color)
axes.fill(theta, label_means, facecolor=label_color, alpha=0.25)
# Legend
legend = axes.legend(legend_labels,
loc=(0.9, .95),
labelspacing=0.1,
fontsize='small')
try:
plt.savefig(options.plot)
except:
logger.log_error(module_name, "Failed to save plot")
logger.log_stop()
#!/usr/bin/python
import os
import multiprocessing
# Enable multithreading for ccx
os.environ['OMP_NUM_THREADS'] = str(multiprocessing.cpu_count())
os.system("cgx -b pre.fbl")
os.system("ccx solve")
os.system("monitor.py solve")
os.system("cgx -b anim.fbl")
os.system("cgx -b post.fbl")
import pandas as pd
import numpy as np
parser = argparse.ArgumentParser(description='Check configs for completeness.')
parser.add_argument(
'-c', '--config', help='Path to config file'
)
parser.add_argument(
'-t', '--type', help='Config type. Valid options are "tr" and "pp"'
)
CONFIG_DEFAULTS = {
'static_data': {
'high_resolution': False,
'cores': multiprocessing.cpu_count(),
'peak_min_width': 0.5,
'peak_min_mz': 3
},
'algorithms': {'XCMS-CW': True, 'XCMS-MF': True},
'default_configs': {
'directory': '../../pp_configs',
'XCMS-CW': 'XCMS-CW_default.INI',
'XCMS-MF': 'XCMS-MF_default.INI'
},
'XCMS_params': {'groupCorr': False},
'merging': {'RT_tol': 0.2}
}
class bcolors:
