for line in range(0,len(str)):
matrix2.append([])
linenow = str[line].split(',')
for elem in range(0,len(linenow)):
matrix2[line].append(int(linenow[elem]))
validmatrix = matrix2 #определяем размерность итоговой матрицы для правильности границ списков в циклах
if len(matrix1) > len(matrix2):
validmatrix = matrix1
if __name__ == '__main__': #если запускается непосредственно данный файл, не в составе подключенной библиотеки. Для работы процессов.
manager = Manager() #менеджер общего пространства памяти процессов
Global = manager.Namespace() #область имен
Global.matrix_result = [0 for i in range(0,len(validmatrix)*len(validmatrix[0]))] #генерируем размеры итоговой матрицы
pool = Pool(len(matrix1)) #создаем пул
pool.starmap(multiply_matrix,[(matrix1,matrix2,linehere,Global) for linehere in range(0,len(matrix1))]) #используем starmap, так как нужно передать несколько аргументов
matrix_result = Global.matrix_result #обновляем переменную в главном процессе
#print result
now = 0
for line in range(0,len(validmatrix)): #выводим результат в консоль
print('[',end='')
def main():
"""Entry point."""
import os
import sys
import gc
from multiprocessing import Process, Manager
from .parser import parse_args
from ..utils.bids import write_derivative_description
parse_args()
popylar = None
if not config.execution.notrack:
import popylar
from ..__about__ import __ga_id__
config.loggers.cli.info(
"Your usage of dmriprep is being recorded using popylar (https://popylar.github.io/). ", # noqa
"For details, see https://nipreps.github.io/dmriprep/usage.html. ",
"To opt out, call dmriprep with a `--notrack` flag",
)
popylar.track_event(__ga_id__, "run", "cli_run")
# CRITICAL Save the config to a file. This is necessary because the execution graph
# is built as a separate process to keep the memory footprint low. The most
# straightforward way to communicate with the child process is via the filesystem.
config_file = config.execution.work_dir / ".dmriprep.toml"
config.to_filename(config_file)
# CRITICAL Call build_workflow(config_file, retval) in a subprocess.
# Because Python on Linux does not ever free virtual memory (VM), running the
# workflow construction jailed within a process preempts excessive VM buildup.
with Manager() as mgr:
from .workflow import build_workflow
retval = mgr.dict()
p = Process(target=build_workflow, args=(str(config_file), retval))
p.start()
p.join()
retcode = p.exitcode or retval.get("return_code", 0)
dmriprep_wf = retval.get("workflow", None)
# CRITICAL Load the config from the file. This is necessary because the ``build_workflow``
# function executed constrained in a process may change the config (and thus the global
# state of dMRIPrep).
config.load(config_file)
if config.execution.reports_only:
sys.exit(int(retcode > 0))
if dmriprep_wf and config.execution.write_graph:
dmriprep_wf.write_graph(graph2use="colored",
format="svg",
simple_form=True)
retcode = retcode or (dmriprep_wf is None) * os.EX_SOFTWARE
if retcode != 0:
sys.exit(retcode)
# Generate boilerplate
with Manager() as mgr:
from .workflow import build_boilerplate
p = Process(target=build_boilerplate,
args=(str(config_file), dmriprep_wf))
p.start()
p.join()
if config.execution.boilerplate_only:
sys.exit(int(retcode > 0))
# Clean up master process before running workflow, which may create forks
gc.collect()
if popylar is not None:
popylar.track_event(__ga_id__, "run", "started")
config.loggers.workflow.log(
15,
"\n".join(["dMRIPrep config:"] +
["\t\t%s" % s for s in config.dumps().splitlines()]),
)
config.loggers.workflow.log(25, "dMRIPrep started!")
errno = 1 # Default is error exit unless otherwise set
try:
dmriprep_wf.run(**config.nipype.get_plugin())
except Exception as e:
if not config.execution.notrack:
popylar.track_event(__ga_id__, "run", "error")
config.loggers.workflow.critical("dMRIPrep failed: %s", e)
raise
else:
config.loggers.workflow.log(25, "dMRIPrep finished successfully!")
# Bother users with the boilerplate only iff the workflow went okay.
if (config.execution.output_dir / "dmriprep" / "logs" /
"CITATION.md").exists():
config.loggers.workflow.log(
25, "Works derived from this dMRIPrep execution should "
"include the following boilerplate: "
f"{config.execution.output_dir / 'dmriprep' / 'logs' / 'CITATION.md'}."
)
if config.workflow.run_reconall:
from templateflow import api
from niworkflows.utils.misc import _copy_any
dseg_tsv = str(
api.get("fsaverage", suffix="dseg", extension=[".tsv"]))
_copy_any(
dseg_tsv,
str(config.execution.output_dir / "dmriprep" /
"desc-aseg_dseg.tsv"),
)
_copy_any(
dseg_tsv,
str(config.execution.output_dir / "dmriprep" /
"desc-aparcaseg_dseg.tsv"),
)
errno = 0
finally:
from niworkflows.reports import generate_reports
from pkg_resources import resource_filename as pkgrf
# Generate reports phase
failed_reports = generate_reports(
config.execution.participant_label,
config.execution.output_dir,
config.execution.run_uuid,
config=pkgrf("dmriprep", "config/reports-spec.yml"),
packagename="dmriprep",
)
write_derivative_description(config.execution.bids_dir,
config.execution.output_dir / "dmriprep")
if failed_reports and not config.execution.notrack:
popylar.track_event(__ga_id__, "run", "reporting_error")
sys.exit(int((errno + failed_reports) > 0))
from pathlib import Path
import os
# TODO: Fix "type: ignore" -> why do I get "Module ... has no attribute ..."?
# from my_garlicsim_py3.garlicsim.general_misc import pickle_tools
from flapy.pickle_tools import dumps_skip, loads_skip # type: ignore # pylint: disable=ungrouped-imports
from flapy.copy_fallback import deepcopy # type: ignore # pylint: disable=ungrouped-imports
from deepdiff import DeepHash # type: ignore # pylint: disable=ungrouped-imports
from flapy.results_parser import FuncDescriptor # type: ignore # pylint: disable=ungrouped-imports
T = TypeVar("T") # pylint: disable=invalid-name
logging.getLogger("deepdiff").setLevel(logging.ERROR)
# Initialize multiprocessing.Manager
MANAGER = Manager()
@contextmanager
def timeout(time):
"""
Usage: `with timeout(5): foo` in order to execute foo with the given timeout
"""
# Register a function to raise a TimeoutError on the signal.
signal.signal(signal.SIGALRM, raise_timeout)
# Schedule the signal to be sent after ``time``.
signal.alarm(time)
try:
yield
finally:
def main(argv=None):
if argv is None:
argv = sys.argv
id_base = int(sys.argv[1])
manager = Manager()
wait_q = PriorityQueue()
out_file = open('latency.csv', 'a')
load_thread = Thread(target=check_load)
load_thread.start()
logger.info("PID: %d" % os.getpid())
logger.info("Running the gauntlet:")
id = id_base
num_clients = 0
for i in range(2):
wait_q.put((time.time(), False, id))
id += 1
num_clients += 1
for i in range(2):
wait_q.put((time.time(), True, id))
id += 1
num_clients += 1
pool = Pool(processes=num_clients)
while True:
t, attacker, id = wait_q.get()
delta = t - time.time()
delay = max(delta, 0)
if delay > 0:
Timer(delay, pool.apply_async, (fetch, (attacker, id))).start()
else:
pool.apply_async(fetch, (attacker, id))
wait_q.task_done()
while not wait_q_q.empty():
wait_q.put(wait_q_q.get())
# we have to make sure there is at least one element in wait_q before the end of the loop
if wait_q.empty():
r = wait_q_q.get()
wait_q.put(r)
while not res_q.empty():
id, n, t, error = res_q.get()
logger.info("id: %d got a response for n = %d in %f error: %d" %
(id, n, t, error))
out_file.write("%d, %d, %f, %d\n" % (id, n, t, error))
#if time.time() - start_time > 120:
# break
out_file.flush()
while not res_q.empty():
res = res_q.get()
out_file.write("%d, %f, %d\n" % (res[0], res[1], res[2]))
return 0
def evolve(self):
"""
the main method, that runs the evolutionary algorithm
"""
# generate the population if starting
if self.started_index == 0:
self.generate_initial_population()
cores_num = cpu_count()
loop_length = int(self.iterations / cores_num)
i = self.started_index
with click.progressbar(length=loop_length,
label='Evolving...') as progressbar:
while i < loop_length:
with Manager() as manager:
people = manager.list([])
workers = []
def get_baby(people):
"""
:param people:
"""
# let's make a baby together LOL
baby = self.make_love()
# let's mutate baby's genes, who knows, maybe we create a x-man or something
baby = self.mutate(baby)
people.append(baby)
for _ in range(cores_num):
w = Process(target=get_baby, args=[people])
w.start()
workers.append(w)
for w in workers:
w.join()
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table.key_value(
[[
'started at',
jh.get_arrow(self.start_time).humanize()
],
[
'index/total', '{}/{}'.format(
(i + 1) * cores_num, self.iterations)
],
[
'population_size, solution_len', '{}, {}'.format(
self.population_size, self.solution_len)
],
[
'route', '{}, {}, {}, {}'.format(
router.routes[0].exchange,
router.routes[0].symbol,
router.routes[0].timeframe,
router.routes[0].strategy_name)
]],
'info',
alignments=('left', 'right'))
print('\n')
# print fittest individuals
fittest_list = [
['rank', 'DNA', 'fitness', 'training|testing logs'],
]
if self.population_size > 50:
number_of_ind_to_show = 25
elif self.population_size > 20:
number_of_ind_to_show = 20
elif self.population_size > 9:
number_of_ind_to_show = 9
else:
raise ValueError(
'self.population_size cannot be less than 10')
for j in range(number_of_ind_to_show):
fittest_list.append([
j + 1, self.population[j]['dna'],
self.population[j]['fitness'],
self.population[j]['log']
], )
table.multi_value(fittest_list,
with_headers=True,
alignments=('left', 'left', 'right',
'left'))
# one person has to die and be replaced with the newborn baby
for baby in people:
random_index = randint(0, len(self.population) - 1)
try:
self.population[random_index] = baby
except IndexError:
print('=============')
print('self.population_size: {}'.format(
self.population_size))
print('self.population length: {}'.format(
len(self.population)))
jh.terminate_app()
self.population = list(
sorted(self.population,
key=lambda x: x['fitness'],
reverse=True))
# reaching the fitness goal could also end the process
if baby['fitness'] >= self.fitness_goal:
progressbar.update(self.iterations - i)
print('\n')
print('fitness goal reached after iteration {}'.
format(i))
return baby
# save progress after every n iterations
if i != 0 and int(i * cores_num) % 50 == 0:
self.save_progress(i)
# store a take_snapshot of the fittest individuals of the population
if i != 0 and i % int(100 / cores_num) == 0:
self.take_snapshot(i * cores_num)
i += 1
print('\n\n')
print('Finished {} iterations.'.format(self.iterations))
return self.population
def _calc_output(self,
place,
parallel=False,
no_check_set=None,
loss=None,
enable_inplace=None,
for_inplace_test=False):
program = Program()
block = program.global_block()
op = self._append_ops(block)
inputs = self._get_inputs(block)
outputs = self._get_outputs(block)
feed_map = self.feed_var(inputs, place)
if for_inplace_test:
# Some variables' tensors hold no buffer (tensor's _holder is NULL), like XShape in reshape2 op,
# and the shapes of those variables contain 0 (eg. Xshape.shape = [0, 2, 5]).
# Set persistable for those variables in order to get them from global_scope for inplace grad test directly other than feed them,
# since feed op calls check_memory_size() which fails when tensor's holder_ is NULL.
for name in op.output_arg_names:
var = block.var(name)
var.persistable = True
original_program = program
#if parallel:
# use_cuda = False
# if isinstance(place, fluid.CUDAPlace):
# use_cuda = True
# compiled_prog = fluid.CompiledProgram(program).with_data_parallel(
# loss_name=loss.name if loss else None, places=place)
# program = compiled_prog
fetch_list = getattr(self, "fetch_list", [])
# if the fetch_list is customized by user, we use it directly.
# if not, fill the fetch_list by the user configured outputs in test.
if len(fetch_list) == 0:
for var_name, var in six.iteritems(outputs):
if no_check_set is not None and var_name in no_check_set:
continue
if isinstance(var, list):
for v in var:
fetch_list.append(v.name)
else:
fetch_list.append(var.name)
# if the fetch_list still empty, fill the fetch_list by the operator output.
if len(fetch_list) == 0:
for out_name, out_dup in Operator.get_op_outputs(self.op_type):
fetch_list.append(str(out_name))
if enable_inplace is not None:
build_strategy = fluid.BuildStrategy()
build_strategy.enable_inplace = enable_inplace
compiled_prog = fluid.CompiledProgram(program).with_data_parallel(
build_strategy=build_strategy, places=place)
program = compiled_prog
# Manager() can not store LoDTensor directly
# So, use one additional element to store output lod
return_results = [Manager().list() for _ in range(len(fetch_list) + 1)]
def closure(**kwargs):
role = kwargs['role']
pfl_mpc.init("aby3", role, "localhost", self.server,
int(self.port))
#init_op = fluid.default_main_program().global_block().ops[0]
#_insert_init_op(program, init_op)
executor = Executor(place)
executor.run()
outs = executor.run(program,
feed=feed_map,
fetch_list=fetch_list,
return_numpy=False)
lod = []
for idx in range(len(fetch_list)):
return_results[idx].append(np.array(outs[idx]))
lod_i = outs[idx].lod()
lod_concat = []
for i in lod_i:
lod_concat.append(i)
lod.append(lod_concat)
return_results[len(fetch_list)].append(lod)
ret = self.multi_party_run(target=closure)
self.assertEqual(ret[0], True)
outs = []
lod = np.array(return_results[len(fetch_list)])
for idx in range(len(fetch_list)):
t = fluid.LoDTensor()
reveal_data = aby3.reconstruct(np.array(return_results[idx]))
t.set(reveal_data, place)
lod_idx = lod[0][idx]
try:
t.set_lod(lod_idx)
except Exception as e:
pass
outs.append(t)
self.op = op
self.program = original_program
if for_inplace_test:
return outs, fetch_list, feed_map, original_program, op.desc
else:
return outs, fetch_list
Ctr = 0
values = ""
BallsCnt = len(vB)
while (Ctr < 5):
rgen = randint(1, int(BallsCnt)) - 1
rval = vB.pop(rgen)
Res[Ctr] = rval
values = values + " " + str(rval)
BallsCnt = BallsCnt - 1
Ctr = Ctr + 1
time.sleep(1)
while (tickets <= args.tickets):
with Manager() as manager:
Balls = manager.list(range(1, 71))
MBalls = manager.list(range(1, 26))
Res = manager.list(range(1, 7))
p1 = Process(target=shuffle_balls, args=(Balls, ))
p1.start()
p2 = Process(target=shuffle_balls, args=(MBalls, ))
p2.start()
time.sleep(2)
p = Process(target=popb, args=(
Balls,
Res,
))
p.start()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='ECT for face alignment')
parser.add_argument('--gpus', default=None, type=int, help='specify the gpu ID')
parser.add_argument('--imgDir', default='../imgs/', type=str, help='path to test images')
parser.add_argument('--outDir', default='../output/', type=str, help='path for saving prediction results')
parser.add_argument('--prototxt', default='../caffe/models/300w/matlab.prototxt', type=str,
help='path to caffe model prototxt')
parser.add_argument('--model', default='../model_data/300w_68pt.caffemodel', type=str,
help='path to the pre-trained caffe model')
parser.add_argument('--verbose', default=True, help='show the landmark prediction results')
parser.add_argument('--nIter', default=5, type=int, help='number of iterations for the turning step')
parser.add_argument('--nComponent', default=30, type=int, help='number of PDM components to be used')
q = Manager().Queue()
po = Pool()
lock = Manager().Lock()
dir_list = get_img_dir_list(parser.parse_args().imgDir)
args = parser.parse_args()
length = len(dir_list)
if (0 == args.gpus):
start = 0
end = length / 2
else:
start = length / 2 - 1
end = length
for i in range(start, end):
def recluster(self, request, pk):
#need to add error handling and resilence
start = time.time()
# print ('run',pk)
clusters = self.request.query_params.get('clusters')
# print ('clusters',clusters)
if clusters is not None:
if int(clusters)<=0:
return Response({'error':'Not a valid number of clusters'},status=500)
#files to be used by the GOUtil
run_obj = Run.objects.get(id=pk)
run_uuid = run_obj.name
genefile_name = 'useruploads/inputgenes-' + run_uuid + '.txt'
enrich_outputfile_name = 'useruploads/enrichment-' + run_uuid + '.txt'
sim_outputfile_name = 'useruploads/funsim-' + run_uuid + '.txt'
semsim_outputfile_name = 'useruploads/semsim-' + run_uuid + '.txt'
clusters_outputfile_name = 'useruploads/clusters-' + run_uuid + '.txt'
###### Data Processing Pipeline ######
try:
clusters_outputfile = open(clusters_outputfile_name, 'w').close()#clear the file
except IOError:
print "Error cluster file doesn't exist"
return Response({'error': 'You must run an enrichment before clustering.'},status=500)
#spectral clustering
spectral_start = time.time()
subprocess.call(['python','/GOUtil/spectralClustering.py', semsim_outputfile_name, clusters_outputfile_name, clusters])
print "Spectral run time %s" % (time.time()-spectral_start)
try:
clusters_outputfile = open(clusters_outputfile_name, 'r')
except IOError:
print "Error no clusters exist"
return Response({'error': 'No clusters found for these Genes and Organism.'},status=500)
#Multi-threaded loader to update the cluster data
load_start = time.time()
manager = Manager()
lock = manager.Lock()
taskworker = partial(loadClustersWorker, lock)
pool = Pool(5)
tokens = [(run_obj.id, line) for line in clusters_outputfile]
pool.map(taskworker, tokens)
pool.close()
pool.join()
print "Multi-threading Loading time: %s" % (time.time()-load_start)
# single_load_start = time.time()
# for (enrichmentrun, enrichmentinfo) in tokens:
# token = enrichmentinfo.split('\t')
# enrichment = Enrichment.objects.get(run__id=enrichmentrun,term__termid=token[0])
# enrichment.cluster = int(token[1])
# enrichment.medoid = 'True'==token[2].replace('\n','')
# enrichment.save()
# print "Single Threaded Loading time: %s" % (time.time()-single_load_start)
db.connection.close()
serializer = RunIncludesSerializer(run_obj)
#cleanup files
clusters_outputfile.close()
print "Overall Loading time: %s" % (time.time()-start)
return Response(serializer.data)
else:
return Response({'error':'Not a valid number of clusters'},status=500)
def generate_initial_population(self):
"""
generates the initial population
"""
cores_num = cpu_count()
loop_length = int(self.population_size / cores_num)
with click.progressbar(
length=loop_length,
label='Generating initial population...') as progressbar:
for i in range(loop_length):
people = []
with Manager() as manager:
dna_bucket = manager.list([])
workers = []
def get_fitness(dna, dna_bucket):
try:
fitness_score, fitness_log = self.fitness(dna)
dna_bucket.append(
(dna, fitness_score, fitness_log))
except Exception as e:
proc = os.getpid()
logger.error('process failed - ID: {}'.format(
str(proc)))
logger.error("".join(
traceback.TracebackException.from_exception(
e).format()))
raise e
try:
for _ in range(cores_num):
dna = ''.join(
choices(self.charset, k=self.solution_len))
w = Process(target=get_fitness,
args=(dna, dna_bucket))
w.start()
workers.append(w)
# join workers
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(
'a process exited with exitcode: {}'.
format(str(w.exitcode)))
except KeyboardInterrupt:
print(jh.color('Terminating session...', 'red'))
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
for d in dna_bucket:
people.append({
'dna': d[0],
'fitness': d[1],
'log': d[2]
})
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [
['Started at',
jh.get_arrow(self.start_time).humanize()],
[
'Index', '{}/{}'.format(len(self.population),
self.population_size)
],
[
'errors/info', '{}/{}'.format(len(store.logs.errors),
len(store.logs.info))
],
[
'Trading Route',
'{}, {}, {}, {}'.format(router.routes[0].exchange,
router.routes[0].symbol,
router.routes[0].timeframe,
router.routes[0].strategy_name)
],
# TODO: add generated DNAs?
# ['-'*10, '-'*10],
# ['DNA', people[0]['dna']],
# ['fitness', round(people[0]['fitness'], 6)],
# ['training|testing logs', people[0]['log']],
]
if jh.is_debugging():
table_items.insert(
3, ['Population Size', self.population_size])
table_items.insert(3, ['Iterations', self.iterations])
table_items.insert(3,
['Solution Length', self.solution_len])
table_items.insert(3, ['-' * 10, '-' * 10])
table.key_value(table_items,
'Optimize Mode',
alignments=('left', 'right'))
# errors
if jh.is_debugging() and len(report.errors()):
print('\n')
table.key_value(report.errors(), 'Error Logs')
for p in people:
self.population.append(p)
# sort the population
self.population = list(
sorted(self.population, key=lambda x: x['fitness'], reverse=True))
def train(epochs, policy):
dir = os.path.abspath(os.getcwd())
print("Current Working Directory: " + dir)
if not os.path.exists(dir + "/tmp"):
os.makedirs(dir + "/tmp")
manager = Manager()
rewardDict = manager.dict()
ns = manager.Namespace()
ns.rewardCount = 0
file = "PSR/train/setting/PacMan.json"
if (policy == "fitted_Q"):
Parameter.edit(file=file, param="algorithm", newval="fitted_Q")
elif (policy == "DRL"):
Parameter.edit(file=file, param="algorithm", newval="DRL")
else:
print(
"Please check the policy input, fitted_Q or DRL. fitted_Q will be set as default."
)
Parameter.edit(file=file, param="algorithm", newval="fitted_Q")
Parameter.readfile(file=file)
print("Learning algorithm / Policy: " + Parameter.algorithm)
RandomSamplingForPSR = True
isbuiltPSR = True
game = PacMan()
game.calulateMaxTestID()
Parameter.maxTestID = game.maxTestID
trainData = TrainingData()
iterNo = 0
agent = Agent(PnumActions=game.getNumActions(),
epsilon=Parameter.epsilon,
inputDim=(Parameter.svdDim, ),
algorithm=Parameter.algorithm,
Parrallel=True)
rdict = readMemoryfromdisk(file="PSR/rewardDict.txt")
copyRewardDict(rewardDict=rewardDict, rewardDict1=rdict)
psrModel = CompressedPSR(game.getGameName())
psrPool = Pool(Parameter.threadPoolSize,
initializer=init,
initargs=(
Parameter.maxTestID,
file,
Lock(),
))
print("Finishing Preparation!")
loadCheckPoint(trainData=trainData,
epoch=iterNo,
psrModel=psrModel,
rewardDict=rewardDict)
trainData = trainData.MergeAllBatchData()
trainSet = None
while iterNo < epochs:
print("Starting Iteration: " + str(iterNo + 1))
if RandomSamplingForPSR:
trainData.newDataBatch()
game.SimulateTrainData(runs=Parameter.runsForCPSR,
isRandom=True,
psrModel=psrModel,
trainData=trainData,
epoch=iterNo - 1,
pool=psrPool,
RunOnVirtualEnvironment=False,
name=game.getGameName(),
rewardDict=rewardDict,
ns=ns)
psrModel.validActObset = trainData.validActOb
WriteEvalUateDataForPacMan(
EvalData=trainData.data[trainData.getBatch()], epoch=-1)
trainData.WriteData(file=dir + "/RandomSampling" + str(iterNo) +
".txt")
RandomSamplingForPSR = False
if isbuiltPSR:
psrModel.build(data=trainData,
aos=trainData.validActOb,
pool=psrPool,
rewardDict=rewardDict)
psrModel.saveModel(epoch=iterNo)
from bin.Util import writeMemoryintodisk
writeMemoryintodisk(file="PSR/rewardDict.txt", data=rewardDict.copy())
print("Convert sampling data into training forms")
if trainSet is None:
trainSet = ConvertToTrainSet(data=trainData,
RewardDict=rewardDict,
pool=psrPool,
epoch=iterNo,
name=game.getGameName(),
psrModel=psrModel)
else:
trainSet = trainSet + ConvertLastBatchToTrainSet(
data=trainData,
RewardDict=rewardDict,
pool=psrPool,
epoch=iterNo,
name=game.getGameName(),
psrModel=psrModel)
print("Starting training")
tick1 = time.time()
print("Iteration: %d/%d" % (iterNo + 1, epochs))
agent.Train_And_Update(data=trainSet, epoch=iterNo, pool=psrPool)
tick2 = time.time()
print("The time spent on training: " + str(tick2 - tick1) + "s")
agent.SaveWeight(epoch=iterNo)
print("Evaluating the agent")
tick3 = time.time()
EvalData = game.SimulateTestingRun(runs=Parameter.testingRuns,
epoch=iterNo,
pool=psrPool,
psrModel=psrModel,
name=game.getGameName(),
rewardDict=rewardDict,
ns=ns)
tick4 = time.time()
print("The time spent on evaluating agent: " + str(tick4 - tick3) +
"s")
trainData.newDataBatch()
game.SimulateTrainData(
runs=Parameter.runsForLearning,
psrModel=psrModel,
trainData=trainData,
isRandom=False,
epoch=iterNo,
pool=psrPool,
RunOnVirtualEnvironment=Parameter.trainingOnVirtualEnvironment,
name=game.getGameName(),
rewardDict=rewardDict,
ns=ns)
trainData.WriteData(file=dir + "/observations/epsilonGreedySampling" +
str(iterNo) + ".txt")
WriteEvalUateDataForPacMan(EvalData=EvalData, epoch=iterNo)
iterNo = iterNo + 1
def evolve(self):
"""
the main method, that runs the evolutionary algorithm
"""
# generate the population if starting
if self.started_index == 0:
self.generate_initial_population()
if len(self.population) < 0.5 * self.population_size:
raise ValueError(
'Too many errors: less then half of the planned population size could be generated.'
)
cores_num = cpu_count()
loop_length = int(self.iterations / cores_num)
i = self.started_index
with click.progressbar(length=loop_length,
label='Evolving...') as progressbar:
while i < loop_length:
with Manager() as manager:
people = manager.list([])
workers = []
def get_baby(people):
try:
# let's make a baby together LOL
baby = self.make_love()
# let's mutate baby's genes, who knows, maybe we create a x-man or something
baby = self.mutate(baby)
people.append(baby)
except Exception as e:
proc = os.getpid()
logger.error('process failed - ID: {}'.format(
str(proc)))
logger.error("".join(
traceback.TracebackException.from_exception(
e).format()))
raise e
try:
for _ in range(cores_num):
w = Process(target=get_baby, args=[people])
w.start()
workers.append(w)
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(
'a process exited with exitcode: {}'.
format(str(w.exitcode)))
except KeyboardInterrupt:
print(jh.color('Terminating session...', 'red'))
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [[
'Started At',
jh.get_arrow(self.start_time).humanize()
],
[
'Index/Total',
'{}/{}'.format((i + 1) * cores_num,
self.iterations)
],
[
'errors/info',
'{}/{}'.format(len(store.logs.errors),
len(store.logs.info))
],
[
'Route', '{}, {}, {}, {}'.format(
router.routes[0].exchange,
router.routes[0].symbol,
router.routes[0].timeframe,
router.routes[0].strategy_name)
]]
if jh.is_debugging():
table_items.insert(3, [
'Population Size, Solution Length',
'{}, {}'.format(self.population_size,
self.solution_len)
])
table.key_value(table_items,
'info',
alignments=('left', 'right'))
# errors
if jh.is_debugging() and len(report.errors()):
print('\n')
table.key_value(report.errors(), 'Error Logs')
print('\n')
print('Best DNA candidates:')
print('\n')
# print fittest individuals
fittest_list = [
[
'Rank', 'DNA', 'Fitness',
'Training log || Testing log'
],
]
if self.population_size > 50:
number_of_ind_to_show = 15
elif self.population_size > 20:
number_of_ind_to_show = 10
elif self.population_size > 9:
number_of_ind_to_show = 9
else:
raise ValueError(
'self.population_size cannot be less than 10')
for j in range(number_of_ind_to_show):
fittest_list.append([
j + 1, self.population[j]['dna'],
self.population[j]['fitness'],
self.population[j]['log']
], )
table.multi_value(fittest_list,
with_headers=True,
alignments=('left', 'left', 'right',
'left'))
# one person has to die and be replaced with the newborn baby
for baby in people:
random_index = randint(0, len(self.population) - 1)
try:
self.population[random_index] = baby
except IndexError:
print('=============')
print('self.population_size: {}'.format(
self.population_size))
print('self.population length: {}'.format(
len(self.population)))
jh.terminate_app()
self.population = list(
sorted(self.population,
key=lambda x: x['fitness'],
reverse=True))
# reaching the fitness goal could also end the process
if baby['fitness'] >= self.fitness_goal:
progressbar.update(self.iterations - i)
print('\n')
print('fitness goal reached after iteration {}'.
format(i))
return baby
# save progress after every n iterations
if i != 0 and int(i * cores_num) % 50 == 0:
self.save_progress(i)
# store a take_snapshot of the fittest individuals of the population
if i != 0 and i % int(100 / cores_num) == 0:
self.take_snapshot(i * cores_num)
i += 1
print('\n\n')
print('Finished {} iterations.'.format(self.iterations))
return self.population
board.getMessageFromPlayer(listOfPlayer)
if board.playerLost() or len(deck_shared_memory) == 0:
board.sendMessageToPlayers("everyone_looses", listOfPlayer)
print(" C'est terminé ! ")
sys.exit(0)
if __name__ == "__main__":
key = 667
lock = threading.Lock()
mq = sysv_ipc.MessageQueue(key, sysv_ipc.IPC_CREAT)
BUFFER_SIZE = 100
deck_shared_memory = Manager().list()
game_shared_memory = Manager().list()
lock = Lock()
mqTypeBoard = 1
os.system("clear")
player_nb = int(input("combien de joueurs ?"))
process_pere = Process(target=board,
args=(mq, mqTypeBoard, game_shared_memory,
deck_shared_memory, lock,
[i + 2 for i in range(player_nb)]))
process_pere.start()
mq.send("creation_jeu".encode(), type=1)
def fill_skeleton_with_model_threaded_new(
self,
nodes,
partition=False,
augment=False,
bounds_input_file=None,
bias=True,
move_batch_size=1,
max_moves=None,
remask_interval=None,
sparse=False,
moves=None,
num_workers=1,
worker_prequeue=1,
reject_early_termination=False,
reject_non_seed_components=True,
region_shape=CONFIG.model.input_fov_shape,
):
"""
Floodfill small regions around a list of seed points.
Necessary inputs:
- model_file
- seeds
- volume
- config
returns Dict[seed: Mask]
"""
self.bias = bias
self.move_batch_size = move_batch_size
self.max_moves = max_moves
self.remask_interval = remask_interval
self.reject_non_seed_components = reject_non_seed_components
# Get Volume
self.volume = self.volume.downsample(CONFIG.volume.resolution)
# Seeds come in real coordinates
node_ids, seeds = zip(*[(
nid,
np.array(
self.volume.world_coord_to_local(
self.volume.real_coord_to_world(seed))),
) for nid, seed in nodes.items()])
map_back = {tuple(seed): nid for seed, nid in zip(seeds, node_ids)}
logger.warning(
"CONFIG input_fov_shape: {} vs Sarbor fov_shape: {}".format(
CONFIG.model.input_fov_shape,
(
region_shape,
CONFIG.volume.resolution,
region_shape / CONFIG.volume.resolution,
),
))
region_shape = region_shape // CONFIG.volume.resolution
pbar = tqdm(desc="Seed queue",
total=len(seeds),
miniters=1,
smoothing=0.0)
num_nodes = len(seeds)
seed_generator = iter(seeds)
manager = Manager()
# Queue of seeds to be picked up by workers.
seed_queue = manager.Queue()
# Queue of results from workers.
results_queue = manager.Queue()
# Dequeue of seeds that were put in seed_queue but have not yet been
# combined by the main process.
dispatched_seeds = deque()
# Seeds that were placed in seed_queue but subsequently covered by other
# results before their results have been processed. This allows workers to
# abort working on these seeds by checking this list.
revoked_seeds = manager.list()
# Results that have been received by the main process but have not yet
# been combined because they were not received in the dispatch order.
unordered_results = {}
final_results = {}
def queue_next_seed():
total = 0
for seed in seed_generator:
if unordered_results.get(tuple(seed)) is not None:
# This seed has already been filled.
total += 1
continue
dispatched_seeds.append(seed)
seed_queue.put(seed)
break
return total
for _ in range(min(num_nodes, num_workers * worker_prequeue)):
processed_nodes = queue_next_seed()
pbar.update(processed_nodes)
if "CUDA_VISIBLE_DEVICES" in os.environ:
set_devices = False
num_workers = 1
logger.warn("Environment variable CUDA_VISIBLE_DEVICES is set, " +
"so only one worker can be used.\n" +
"See https://github.com/aschampion/diluvian/issues/11")
else:
set_devices = True
workers = []
loading_lock = manager.Lock()
for worker_id in range(num_workers):
w = Process(
target=self.worker,
args=(
worker_id,
set_devices,
self.volume,
region_shape,
seed_queue,
results_queue,
loading_lock,
revoked_seeds,
),
)
w.start()
workers.append(w)
while dispatched_seeds:
processed_seeds = 1
expected_seed = dispatched_seeds.popleft()
logger.debug("Expecting seed %s", expected_seed)
if tuple(expected_seed) in unordered_results:
logger.debug("Expected seed %s is in old results",
expected_seed)
seed = expected_seed
body = unordered_results[tuple(seed)]
del unordered_results[tuple(seed)] # WHY
else:
seed, body = results_queue.get(True)
processed_seeds += queue_next_seed()
while not np.array_equal(seed, expected_seed):
logger.debug("Node %s is early, stashing", seed)
unordered_results[tuple(seed)] = body
seed, body = results_queue.get(True)
processed_seeds += queue_next_seed()
logger.debug("Processing node at %s", seed)
pbar.update(processed_seeds)
if final_results.get(tuple(seed)) is not None:
# This seed has already been filled.
logger.debug(
"Seed (%s) was filled but has been covered in the meantime.",
np.array_str(seed),
)
loading_lock.acquire()
if tuple(seed) in revoked_seeds:
revoked_seeds.remove(tuple(seed))
loading_lock.release()
continue
if body is None:
logger.warning("Body of Seed ({}) is None".format(seed))
continue
# REDO THIS SEED
raise NotImplementedError
continue
if not body.is_seed_in_mask():
logger.warning("Seed ({}) is not in its body.".format(seed))
continue
# REDO THIS SEED
raise NotImplementedError
continue
mask, bounds = body._get_bounded_mask(
CONFIG.postprocessing.closing_shape)
body_size = np.count_nonzero(mask)
if body_size == 0:
logger.warning("Body of seed {} is empty.".format(seed))
continue
# REDO THIS SEED
raise NotImplementedError
continue
final_results[map_back[tuple(seed)]] = mask
logger.warning("Filled seed ({})".format(seed))
for _ in range(num_workers):
seed_queue.put("DONE")
for wid, worker in enumerate(workers):
worker.join()
manager.shutdown()
pbar.close()
return final_results
def multi_mode(cli_parsed):
dbm = db_manager.DB_Manager(cli_parsed.d + '/ew.db')
dbm.open_connection()
if not cli_parsed.resume:
dbm.initialize_db()
dbm.save_options(cli_parsed)
m = Manager()
targets = m.Queue()
lock = m.Lock()
multi_counter = m.Value('i', 0)
display = None
def exitsig(*args):
dbm.close()
if current_process().name == 'MainProcess':
print('')
print('Resume using ./EyeWitness.py --resume {0}'.format(cli_parsed.d + '/ew.db'))
os._exit(1)
signal.signal(signal.SIGINT, exitsig)
if cli_parsed.resume:
pass
else:
url_list = target_creator(cli_parsed)
if cli_parsed.web:
for url in url_list:
dbm.create_http_object(url, cli_parsed)
if cli_parsed.web:
if cli_parsed.web and not cli_parsed.show_selenium:
display = Display(visible=0, size=(1920, 1080))
display.start()
multi_total = dbm.get_incomplete_http(targets)
if multi_total > 0:
if cli_parsed.resume:
print('Resuming Web Scan ({0} Hosts Remaining)'.format(str(multi_total)))
else:
print('Starting Web Requests ({0} Hosts)'.format(str(multi_total)))
if multi_total < cli_parsed.threads:
num_threads = multi_total
else:
num_threads = cli_parsed.threads
for i in range(num_threads):
targets.put(None)
try:
workers = [Process(target=worker_thread, args=(
cli_parsed, targets, lock, (multi_counter, multi_total))) for i in range(num_threads)]
for w in workers:
w.start()
for w in workers:
w.join()
except Exception as e:
print(str(e))
if display is not None:
display.stop()
results = dbm.get_complete_http()
dbm.close()
m.shutdown()
sort_data_and_write(cli_parsed, results)
def invoke(self, request):
#need to add error handling and resilence
start = time.time()
genes = request.data.get('genes')
pvalue = request.data.get('pvalue')
clusters = request.data.get('clusters')
organism = request.data.get('organism')
background = request.data.get('background')
namespace = request.data.get('namespace')
goontology = request.data.get('goontology')
if genes is not None and pvalue is not None and clusters is not None and goontology is not None:
if organism not in ['hsa','gga','bta','cfa','mmu','rno','cel','ath','dme','sce','eco','dre'] or int(clusters)<=0:
return Response({'error': 'Organism type and/or clusters are not valid parameters'},status=500)
genes = bleach.clean(genes).replace(',', '\n')
background = bleach.clean(background)
goontology = bleach.clean(goontology)
try:
ontology = Ontology.objects.get(pk=int(goontology))
except Ontology.DoesNotExist:
return Response({'error':'Ontology does not exist'},status=404)
#temp files to be used by the GOUtil
tmp_uuid = str(uuid.uuid4())
genefile_name = 'useruploads/inputgenes-' + tmp_uuid + '.txt'
enrich_outputfile_name = 'useruploads/enrichment-' + tmp_uuid + '.txt'
sim_outputfile_name = 'useruploads/funsim-' + tmp_uuid + '.txt'
semsim_outputfile_name = 'useruploads/semsim-' + tmp_uuid + '.txt'
clusters_outputfile_name = 'useruploads/clusters-' + tmp_uuid + '.txt'
genefile = open(genefile_name, 'w+')
genefile.write(genes)
genefile.close()
new_run = Run(name=tmp_uuid,ip=get_ip(request))
new_run.save()
base_dir = '/GOUtildata/'+ontology.name+'/'
annotation_file_name = base_dir+'ann.'+organism+'.'+namespace+'.txt'
edgelist_file_name = base_dir+'edgeList.'+'bp.txt'
if background:
background_file_name = 'useruploads/background-' + tmp_uuid + '.txt'
backgroundfile = open(background_file_name, 'w+')
backgroundfile.write(background)
backgroundfile.close()
else:
background_file_name = base_dir+'background.'+organism+'.'+namespace+'.txt'
###### Enrichment Pipeline ######
#invoke enrichment util to compute enrichments
e_start = time.time()
subprocess.call(['/GOUtil/./enrich', '-a', annotation_file_name, '-e', edgelist_file_name, '-t', genefile_name, '-b', background_file_name, '-o', enrich_outputfile_name, '-p', pvalue])
print "Enrich run time %s" % (time.time()-e_start)
try:
enrich_outputfile = open(enrich_outputfile_name, 'r')
except IOError:
print "Error no enrichment file."
return Response({'error': 'No Enrichment found for these Genes and Organism.'},status=500)
#Multi-threaded loader to load in all of the enrichment terms
enrich_manager = Manager()
enrich_lock = enrich_manager.Lock()
enrich_taskworker = partial(loadEnrichmentsWorker, enrich_lock)
enrich_pool = Pool(8)
enrich_tokens = [(new_run.id, line, ontology.id) for line in enrich_outputfile]
enrich_pool.map(enrich_taskworker, enrich_tokens)
fun_sim_start = time.time()
#invoke funSim util to compute semantic similarity
subprocess.call(['/GOUtil/./funSim', '-a', annotation_file_name, '-e', edgelist_file_name, '-o', sim_outputfile_name, '-t',"AIC", '-f', enrich_outputfile_name])
print "FunSim run time %s" % (time.time()-fun_sim_start)
mds_sim_start = time.time()
#compute x, y coordinates for enriched terms
subprocess.call(['python','/GOUtil/mdsSemSim.py', sim_outputfile_name, semsim_outputfile_name])
print "MDS run time %s" % (time.time()-mds_sim_start)
#Multi-threaded loader to load in all of the enrichment term coordinates
try:
semsim_outputfile = open(semsim_outputfile_name, 'r')
except IOError:
print "Error No MDS file"
return Response({'error': 'Semantic Similarity Computation Failed for these Genes and Organism.'},status=500)
# ensure enrichments are loaded before proceeding
enrich_pool.close()
enrich_pool.join()
coord_manager = Manager()
coord_lock = coord_manager.Lock()
coord_taskworker = partial(loadCoordsWorker, coord_lock)
coord_pool = Pool(5)
coord_tokens = [(new_run.id, line) for line in semsim_outputfile]
coord_pool.map(coord_taskworker, coord_tokens)
coord_pool.close()
coord_pool.join()
#spectral clustering
spectral_start = time.time()
subprocess.call(['python','/GOUtil/spectralClustering.py', semsim_outputfile_name, clusters_outputfile_name, "-1"])
print "Spectral run time %s" % (time.time()-spectral_start)
try:
clusters_outputfile = open(clusters_outputfile_name, 'r')
except IOError:
print "Error no clusters exist"
return Response({'error': 'No clusters found for these Genes and Organism.'},status=500)
#Multi-threaded loader to update the cluster data
cluster_manager = Manager()
cluster_lock = cluster_manager.Lock()
cluster_taskworker = partial(loadClustersWorker, cluster_lock)
cluster_pool = Pool(5)
cluster_tokens = [(new_run.id, line) for line in clusters_outputfile]
cluster_pool.map(cluster_taskworker, cluster_tokens)
cluster_pool.close()
cluster_pool.join()
print "Loading time: %s" % (time.time()-start)
db.connection.close()
serializer = RunSerializer(new_run)
#cleanup temp files
enrich_outputfile.close()
semsim_outputfile.close()
clusters_outputfile.close()
# os.remove(genefile_name)
# os.remove(enrich_outputfile_name)
return Response(serializer.data)
# return Response({'runid': new_run.id})
else:
return Response({'error': 'Invalid parameters'},status=500)
# Send frame to global
write_frame_list[worker_id] = frame_process
# Expect next worker to write frame
Global.write_num = next_id(Global.write_num, worker_num)
if __name__ == '__main__':
Total_time=0
# Fix Bug on MacOS
if platform.system() == 'Darwin':
set_start_method('forkserver')
# Global variables
Global = Manager().Namespace()
Global.buff_num = 1
Global.read_num = 1
Global.write_num = 1
Global.frame_delay = 0
Global.is_exit = False
read_frame_list = Manager().dict()
write_frame_list = Manager().dict()
# Number of workers (subprocess use to process frames)
if cpu_count() > 2:
worker_num = cpu_count() - 1 # 1 for capturing frames
else:
worker_num = 2
# Subprocess list
# 进程池之间的通信
from multiprocessing import Manager, Pool
import time
import random
def write(q):
l = [1, 2, 3]
for i in l:
q.put(i)
def read(q):
for i in range(q.qsize()):
print(q.get())
if __name__ == "__main__":
q = Manager().Queue()
pool = Pool()
pool.apply(write, (q, ))
pool.apply(read, (q, ))
pool.close()
pool.join()
def _get_gradient(self,
input_to_check,
place,
output_names,
no_grad_set,
parallel=False):
prog = Program()
block = prog.global_block()
self._append_ops(block)
# Manager() can not store LoDTensor directly
# So, use one additional element to store output lod
fetch_list = []
fetch_list_len = len(input_to_check)
return_results = [Manager().list() for _ in range(fetch_list_len + 1)]
def closure(**kwargs):
role = kwargs['role']
pfl_mpc.init("aby3", role, "localhost", self.server,
int(self.port))
loss = append_loss_ops(block, output_names)
param_grad_list = append_backward(loss=loss,
parameter_list=input_to_check,
no_grad_set=no_grad_set)
inputs = self._get_inputs(block)
feed_dict = self.feed_var(inputs, place)
fetch_list = [g for p, g in param_grad_list]
executor = Executor(place)
executor.run()
outs = executor.run(prog,
feed=feed_dict,
fetch_list=fetch_list,
return_numpy=False)
# append lod information in last position
lod = []
for idx in range(fetch_list_len):
return_results[idx].append(np.array(outs[idx]))
lod_i = outs[idx].lod()
lod_concat = []
for i in lod_i:
lod_concat.append(i)
lod.append(lod_concat)
return_results[fetch_list_len].append(lod)
ret = self.multi_party_run(target=closure)
self.assertEqual(ret[0], True)
outs = []
lod = np.array(return_results[fetch_list_len])
# from numpy array to LoDTensor
for idx in range(fetch_list_len):
t = fluid.LoDTensor()
reveal_data = aby3.reconstruct(np.array(return_results[idx]))
t.set(reveal_data, place)
lod_idx = lod[0][idx]
# TODO: fix: exception throw because some output lod error in gru op
# out.set_lod(out.lod()) will throw exception
try:
t.set_lod(lod_idx)
except Exception as e:
pass
outs.append(t)
return outs
#-*-coding:utf-8-*-
# Author:Lu Wei
from multiprocessing import Process, Manager, Lock
def func(dic, l):
with l:
dic['count'] -= 1
if __name__ == '__main__':
m = Manager()
l = Lock()
dic = m.dict({'count': 100})
p_l = []
for i in range(10):
p = Process(target=func, args=(dic, l))
p.start()
p_l.append(p)
for p in p_l:
p.join()
print(dic)
def run_wrap_production(self, test=None):
"""
Wrapper function. It assumes the initialisation stage has already
happend Writes XRSL file with the appropiate information and send a
producrun number of jobs to the arc queue
"""
from pyHepGrid.src.header import baseSeed, producRun, jobName, count, memory
# runcard names (keys)
# dCards, dictionary of { 'runcard' : 'name' }
rncards, dCards = util.expandCard()
self.runfolder = header.runcardDir
job_type = "Production"
header.logger.info("Runcards selected: {0}".format(" ".join(
r for r in rncards)))
for r in rncards:
joblist = []
# Check whether this run has something on the gridStorage
self.check_for_existing_output(r, dCards[r])
# use the same unique name for all seeds since
# we cannot multiprocess the arc submission
keyquit = None
# Sanity check for test queue
if test and producRun > 5:
self._press_yes_to_continue(
" \033[93m WARNING:\033[0m About to submit a large "
"number ({0}) of jobs to the test queue.".format(
producRun))
# use iterator for memory reasons :)
from multiprocessing import Manager
# Use shared memory list in case of submission failure
jobids = Manager().list()
arg_sets = self.arg_iterator(r, dCards, jobName, baseSeed,
producRun, test, jobids, count,
memory)
try:
joblist = self._multirun(self.run_single_production,
arg_sets,
n_threads=min(
header.arc_submit_threads,
producRun))
except (Exception, KeyboardInterrupt) as interrupt:
print("\n")
joblist = jobids
header.logger.error(
"Submission error encountered. "
"Inserting all successful submissions to database")
keyquit = interrupt
# Create daily path
pathfolder = util.generatePath(warmup=False)
# Create database entry
jobStr = ' '.join(joblist)
dataDict = {
'jobid': jobStr,
'date': str(datetime.now()),
'pathfolder': pathfolder,
'runcard': r,
'jobtype': job_type,
'runfolder': dCards[r],
'iseed': str(baseSeed),
'no_runs': str(producRun),
'status': "active",
}
if len(joblist) > 0:
self.dbase.insert_data(self.table, dataDict)
# Set jobs to failed status if no jobid returned
dbid = self.get_active_dbids()[-1]
statuses = [
self.cUNK if i != "None" else self.cMISS for i in joblist
]
self._set_new_status(dbid, statuses)
else:
header.logger.critical(
"No jobids returned, no database entry inserted for "
"submission: {0} {1}".format(r, dCards[r]))
if keyquit is not None:
raise keyquit
def runModel(train,
test,
trainTarget,
testTarget,
targetVar=target,
subMode=None,
submission=0):
print("started")
from sklearn.neural_network import MLPClassifier
if subMode is None:
que = Manager().Queue()
pool = Pool(processes=1)
counter = 0
dict1 = {}
L1 = [i for i in range(12, 20)]
L2 = [i for i in range(4, 12)]
for i in L1:
for j in L2:
if j < i and subMode is None:
if j != 0:
mlp = MLPClassifier(hidden_layer_sizes=(i, j),
max_iter=250,
alpha=1e-4,
solver='sgd',
verbose=False,
tol=1e-4,
random_state=1,
learning_rate_init=0.1)
else:
mlp = MLPClassifier(hidden_layer_sizes=(i),
max_iter=250,
alpha=1e-4,
solver='sgd',
verbose=False,
tol=1e-4,
random_state=1,
learning_rate_init=0.1)
pool.apply_async(jugad,
args=(que, mlp, train, trainTarget, i, j))
counter = counter + 1
elif subMode is not None:
mlp = MLPClassifier(hidden_layer_sizes=subMode,
max_iter=250,
alpha=1e-4,
solver='sgd',
verbose=False,
tol=1e-4,
random_state=1,
learning_rate_init=0.1)
mlp.fit(train, trainTarget[target])
trainer = pd.DataFrame(mlp.predict_proba(train.values),
columns=['good', target],
index=train.index)[[target]]
submision = pd.DataFrame(mlp.predict_proba(test.values),
columns=['good', target],
index=test.index)[[target]]
print(trainTarget[[target]].mean(axis=0),
trainer[[target]].mean(axis=0))
#lorenzCurve(trainTarget["TARGET"].values.flatten(),trainer[target].values.flatten())
# rawTest['actual'] = testTarget[target]
# rawTest['TARGET1']=submision[target]
score_test = metrics.roc_auc_score(testTarget[target],
submision[[target]])
score_train = metrics.roc_auc_score(trainTarget[target],
trainer[[target]])
print(score_train, score_test)
#error = logLoss(rawTest[varSelected+['actual', 'TARGET1']], 'actual', 'TARGET1').sort_values(['error'])
#error.to_csv("/home/pooja/PycharmProjects/datanalysis/finalDatasets/error.csv")
return pd.DataFrame(mlp.predict_proba(test.values),
columns=['good', target],
index=testTarget.index)[[target]]
pool.close()
pool.join()
for element in range(counter):
field = que.get()
mlp = field[0]
i = field[1]
j = field[2]
trainer = pd.DataFrame(mlp.predict_proba(train.values),
columns=['good', target],
index=train.index)[[target]]
submision = pd.DataFrame(mlp.predict_proba(test.values),
columns=['good', target],
index=test.index)[[target]]
# submision.index.name='SK_ID_CURR'
# print(submision.shape)
# submision.to_csv("submission.csv")
score_test = metrics.roc_auc_score(testTarget[target],
submision[[target]])
score_train = metrics.roc_auc_score(trainTarget[target],
trainer[[target]])
try:
dict1[str(i) + "_" +
str(j)] = str(score_train) + "_" + str(score_test)
except:
print(i, j)
pass
print("starting")
for key in dict1.keys():
print(key)
print(dict1[key])
return None
def hc(self,
metric='AIC',
max_iter=100,
debug=False,
restriction=None,
whitelist=None):
"""
Greedy Hill Climbing search proceeds by choosing the move
which maximizes the increase in fitness of the
network at the current step. It continues until
it reaches a point where there does not exist any
feasible single move that increases the network fitness.
It is called "greedy" because it simply does what is
best at the current iteration only, and thus does not
look ahead to what may be better later on in the search.
For computational saving, a Priority Queue (python's heapq)
can be used to maintain the best operators and reduce the
complexity of picking the best operator from O(n^2) to O(nlogn).
This works by maintaining the heapq of operators sorted by their
delta score, and each time a move is made, we only have to recompute
the O(n) delta-scores which were affected by the move. The rest of
the operator delta-scores are not affected.
For additional computational efficiency, we can cache the
sufficient statistics for various families of distributions -
therefore, computing the mutual information for a given family
only needs to happen once.
The possible moves are the following:
- add edge
- delete edge
- invert edge
Arguments
---------
*data* : a nested numpy array
The data from which the Bayesian network
structure will be learned.
*metric* : a string
Which score metric to use.
Options:
- AIC
- BIC / MDL
- LL (log-likelihood)
*max_iter* : an integer
The maximum number of iterations of the
hill-climbing algorithm to run. Note that
the algorithm will terminate on its own if no
improvement is made in a given iteration.
*debug* : boolean
Whether to print the scores/moves of the
algorithm as its happening.
*restriction* : a list of 2-tuples
For MMHC algorithm, the list of allowable edge additions.
Returns
-------
*bn* : a BayesNet object
"""
# INITIALIZE NETWORK W/ NO EDGES
# maintain children and parents dict for fast lookups
self.c_dict = dict([(n, []) for n in self.nodes])
self.p_dict = dict([(n, []) for n in self.nodes])
self.restriction = restriction
self.whitelist = whitelist
if whitelist is None:
whitelist = []
for (u, v) in whitelist:
self.c_dict[u].append(v)
self.p_dict[v].append(u)
print("Whitelist", whitelist)
self.bn = BayesNet(self.c_dict)
# COMPUTE INITIAL LIKELIHOOD SCORE
# value_dict = dict([(n, np.unique(np_data[:,i])) for i,n in enumerate(names)])
print("Nodes:", list(self.bn.nodes()))
score = model_score(self.data, self.bn) - model_complexity(
self.bn, self.nrow, metric)
print("Initial Score:", score)
# CREATE EMPIRICAL DISTRIBUTION OBJECT FOR CACHING
#ED = EmpiricalDistribution(data,names)
_iter = 0
improvement = True
man = Manager()
mut_inf_cache = man.dict()
configs_cache = man.dict()
while improvement:
start_t = time.time()
improvement = False
max_delta = 0
max_operation = None
if debug:
print('ITERATION: ', _iter)
return_queue = Queue()
p_add = Process(target=self.test_arc_additions,
args=(configs_cache, mut_inf_cache, return_queue))
p_rem = Process(target=self.test_arc_deletions,
args=(configs_cache, mut_inf_cache, return_queue))
p_rev = Process(target=self.test_arc_reversals,
args=(configs_cache, mut_inf_cache, return_queue))
p_add.start()
p_rem.start()
p_rev.start()
p_add.join()
p_rem.join()
p_rev.join()
while not return_queue.empty():
results = return_queue.get()
if results[1] > max_delta:
max_arc = results[0]
max_delta = results[1]
max_operation = results[2]
max_qi = results[3]
### DETERMINE IF/WHERE IMPROVEMENT WAS MADE ###
if max_operation:
score += max_delta
improvement = True
u, v = max_arc
str_arc = [e for e in max_arc]
if max_operation == 'Addition':
if debug:
print("delta:", max_delta)
print('ADDING: ', str_arc, '\n')
self.p_dict[v].append(u)
self.bn.add_edge(u, v)
self.bn.F[v]["qi"] = max_qi
elif max_operation == 'Deletion':
if debug:
print("delta:", max_delta)
print('DELETING: ', str_arc, '\n')
self.p_dict[v].remove(u)
self.bn.remove_edge(u, v)
self.bn.F[v]["qi"] = max_qi
elif max_operation == 'Reversal':
if debug:
print("delta:", max_delta)
print('REVERSING: ', str_arc, '\n')
self.p_dict[v].remove(u)
self.bn.remove_edge(u, v)
self.bn.F[v]['qi'] = max_qi[1]
self.p_dict[u].append(v)
self.bn.add_edge(v, u)
self.bn.F[u]['qi'] = max_qi[0]
print("Model score:", score
) # TODO: improve so only changed elements get an update
else:
if debug:
print('No Improvement on Iter: ', _iter)
print("Time for iteration:", time.time() - start_t)
### TEST FOR MAX ITERATION ###
_iter += 1
# if _iter > max_iter:
# if debug:
# print('Max Iteration Reached')
# break
bn = BayesNet(self.c_dict)
print("Size of Cache", len(mut_inf_cache))
print("SCORE =", score)
return bn
def run_tests_parallel(tests, prefix, options):
# This queue will contain the results of the various tests run.
# We could make this queue a global variable instead of using
# a manager to share, but this will not work on Windows.
queue_manager = Manager()
async_test_result_queue = queue_manager.Queue()
# This queue will be used by the result process to indicate
# that it has received a result and we can start a new process
# on our end. The advantage is that we don't have to sleep and
# check for worker completion ourselves regularly.
notify_queue = queue_manager.Queue()
# This queue will contain the return value of the function
# processing the test results.
total_tests = len(tests) * options.repeat
result_process_return_queue = queue_manager.Queue()
result_process = Process(target=process_test_results_parallel,
args=(async_test_result_queue,
result_process_return_queue, notify_queue,
total_tests, options))
result_process.start()
# Ensure that a SIGTERM is handled the same way as SIGINT
# to terminate all child processes.
sigint_handler = signal.getsignal(signal.SIGINT)
signal.signal(signal.SIGTERM, sigint_handler)
worker_processes = []
def remove_completed_workers(workers):
new_workers = []
for worker in workers:
if worker.is_alive():
new_workers.append(worker)
else:
worker.join()
return new_workers
try:
testcnt = 0
# Initially start as many jobs as allowed to run parallel
# Always enqueue at least one to avoid a curious deadlock
for i in range(max(1, min(options.max_jobs, total_tests))):
notify_queue.put(True)
# For every item in the notify queue, start one new worker.
# Every completed worker adds a new item to this queue.
while notify_queue.get():
if testcnt < total_tests:
# Start one new worker
test = tests[testcnt % len(tests)]
worker_process = Process(target=wrap_parallel_run_test,
args=(test, prefix,
async_test_result_queue,
options))
worker_processes.append(worker_process)
worker_process.start()
testcnt += 1
# Collect completed workers
worker_processes = remove_completed_workers(worker_processes)
else:
break
# Wait for all processes to terminate
while len(worker_processes) > 0:
worker_processes = remove_completed_workers(worker_processes)
# Signal completion to result processor, then wait for it to complete
# on its own
async_test_result_queue.put(None)
result_process.join()
# Return what the result process has returned to us
return result_process_return_queue.get()
except (Exception, KeyboardInterrupt) as e:
# Print the exception if it's not an interrupt,
# might point to a bug or other faulty condition
if not isinstance(e, KeyboardInterrupt):
traceback.print_exc()
for worker in worker_processes:
try:
worker.terminate()
except:
pass
result_process.terminate()
return False
def generate_initial_population(self):
"""
generates the initial population
"""
cores_num = cpu_count()
loop_length = int(self.population_size / cores_num)
with click.progressbar(
length=loop_length,
label='Generating initial population...') as progressbar:
for i in range(loop_length):
people = []
with Manager() as manager:
dna_bucket = manager.list([])
workers = []
def get_fitness(dna, dna_bucket):
"""
:param dna:
:param dna_bucket:
"""
fitness_score, fitness_log = self.fitness(dna)
dna_bucket.append((dna, fitness_score, fitness_log))
for _ in range(cores_num):
dna = ''.join(
choices(self.charset, k=self.solution_len))
w = Process(target=get_fitness, args=(dna, dna_bucket))
w.start()
workers.append(w)
# join workers
for w in workers:
w.join()
for d in dna_bucket:
people.append({
'dna': d[0],
'fitness': d[1],
'log': d[2]
})
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table.key_value([
['started at',
jh.get_arrow(self.start_time).humanize()],
[
'index/total', '{}/{}'.format(len(self.population),
self.population_size)
],
['-', '-'],
['population_size', self.population_size],
['iterations', self.iterations],
['solution_len', self.solution_len],
[
'route', '{}, {}, {}, {}'.format(
router.routes[0].exchange, router.routes[0].symbol,
router.routes[0].timeframe,
router.routes[0].strategy_name)
],
['-', '-'],
['DNA', people[0]['dna']],
['fitness', round(people[0]['fitness'], 6)],
['training|testing logs', people[0]['log']],
],
'baby',
alignments=('left', 'right'))
for p in people:
self.population.append(p)
# sort the population
self.population = list(
sorted(self.population, key=lambda x: x['fitness'], reverse=True))
taskQueue.close()
#taskQueue.join()
logger = multiprocessing.log_to_stderr()
logger.setLevel(logging.WARNING)
def error(msg, *args):
return logger.error(msg, *args)
# start (in parallel)!
nCores = max(1, multiprocessing.cpu_count()-1)
#print(nCores)
with ProcessPoolExecutor(max_workers=nCores) as workers, Manager() as manager:
taskQueue = JoinableQueue()
result = workers.submit(run)
while not result.done():
taskArgs = taskQueue.get(block=True)
if isinstance(taskArgs, Iterable):
task, *args = taskArgs
task(*args)
else:
taskArgs()
#run()
print('Done!')
sleep(2)
def run(self): # call start()时 就会调用run(run为单进程).
while True:
item = self.queue.get()
self.horizontal_mirror_imgs(self.args['imgs_path'],self.args['xml_path'],item,self.args['save_path'])
print("Consumer-->%s" % item, self.label)
self.queue.task_done()
if __name__ == '__main__':
imgs_path = '/home/lzc274500/WorkSpace/ZOUZHEN/Pytorch/crnn_chinese_characters_rec/data/vehicle/Image'
xml_path = '/home/lzc274500/WorkSpace/ZOUZHEN/Pytorch/crnn_chinese_characters_rec/data_generator/data_set/PK5HD00000002.xml'
save_path = '/home/lzc274500/WorkSpace/ZOUZHEN/Pytorch/crnn_chinese_characters_rec/data_generator/data_set/save_path'
label_path = '/home/lzc274500/WorkSpace/ZOUZHEN/Pytorch/crnn_chinese_characters_rec/data_generator/data_set/train.txt'
pathlist = os.listdir(imgs_path)
# 统计计算内部的核心进程数
if not os.path.exists(save_path):
os.makedirs(save_path)
cores = multiprocessing.cpu_count()
qMar = Manager()
# 取核心进程数的一半建立数据队列
q1 = qMar.Queue(cores-5)
p = Producer(q1, pathlist)
processes = []
processes.append(p)
# print(int(cores/2))
for i in range(cores-5):
processes.append(Consumer(q1,i,label_path,imgs_path=imgs_path,xml_path=xml_path, save_path=save_path))
[process.start() for process in processes]
[process.join() for process in processes]
async def schedule_formatting(
sources: Set[Path],
fast: bool,
write_back: WriteBack,
mode: Mode,
report: "Report",
loop: asyncio.AbstractEventLoop,
executor: Executor,
) -> None:
"""Run formatting of `sources` in parallel using the provided `executor`.
(Use ProcessPoolExecutors for actual parallelism.)
`write_back`, `fast`, and `mode` options are passed to
:func:`format_file_in_place`.
"""
cache: Cache = {}
if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
cache = read_cache(mode)
sources, cached = filter_cached(cache, sources)
for src in sorted(cached):
report.done(src, Changed.CACHED)
if not sources:
return
cancelled = []
sources_to_cache = []
lock = None
if write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
# For diff output, we need locks to ensure we don't interleave output
# from different processes.
manager = Manager()
lock = manager.Lock()
tasks = {
asyncio.ensure_future(
loop.run_in_executor(
executor, format_file_in_place, src, fast, mode, write_back, lock
)
): src
for src in sorted(sources)
}
pending = tasks.keys()
try:
loop.add_signal_handler(signal.SIGINT, cancel, pending)
loop.add_signal_handler(signal.SIGTERM, cancel, pending)
except NotImplementedError:
# There are no good alternatives for these on Windows.
pass
while pending:
done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
for task in done:
src = tasks.pop(task)
if task.cancelled():
cancelled.append(task)
elif task.exception():
report.failed(src, str(task.exception()))
else:
changed = Changed.YES if task.result() else Changed.NO
# If the file was written back or was successfully checked as
# well-formatted, store this information in the cache.
if write_back is WriteBack.YES or (
write_back is WriteBack.CHECK and changed is Changed.NO
):
sources_to_cache.append(src)
report.done(src, changed)
if cancelled:
await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)
if sources_to_cache:
write_cache(cache, sources_to_cache, mode)
def consumer_task(q, fibo_dict):
while not q.empty():
value = q.get(True, 0.05)
a, b = 0, 1
for item in range(value):
a, b = b, a + b
fibo_dict[value] = a
logger.info("consumer [%s] getting value [%d] from queue..." %
(current_process().name, value))
if __name__ == "__main__":
data_queue = Queue()
number_of_cpus = cpu_count()
manager = Manager()
logger.info("cpu core count: " + str(number_of_cpus))
# 프로세스의 최종 결과를 삽입.
fibo_dict = manager.dict()
producer = Process(target=producer_task, args=(data_queue, fibo_dict))
producer.start()
producer.join()
consumer_list = []
for i in range(number_of_cpus):
consumer = Process(target=consumer_task, args=(data_queue, fibo_dict))
consumer.start()
consumer_list.append(consumer)
def _executors_repro(
self, executors: dict, jobs: Optional[int] = 1
) -> Mapping[str, Mapping[str, str]]:
"""Run dvc repro for the specified BaseExecutors in parallel.
Returns:
dict mapping stash revs to the successfully executed experiments
for each stash rev.
"""
result: Dict[str, Dict[str, str]] = defaultdict(dict)
manager = Manager()
pid_q = manager.Queue()
rel_cwd = relpath(os.getcwd(), self.repo.root_dir)
with ProcessPoolExecutor(max_workers=jobs) as workers:
futures = {}
for rev, executor in executors.items():
future = workers.submit(
executor.reproduce,
executor.dvc_dir,
rev,
queue=pid_q,
name=executor.name,
rel_cwd=rel_cwd,
log_level=logger.getEffectiveLevel(),
)
futures[future] = (rev, executor)
try:
wait(futures)
except KeyboardInterrupt:
# forward SIGINT to any running executor processes and
# cancel any remaining futures
pids = {}
while not pid_q.empty():
rev, pid = pid_q.get()
pids[rev] = pid
for future, (rev, _) in futures.items():
if future.running():
os.kill(pids[rev], signal.SIGINT)
elif not future.done():
future.cancel()
for future, (rev, executor) in futures.items():
rev, executor = futures[future]
try:
exc = future.exception()
if exc is None:
exec_result = future.result()
result[rev].update(
self._collect_executor(executor, exec_result)
)
elif not isinstance(exc, CheckpointKilledError):
logger.error(
"Failed to reproduce experiment '%s'", rev[:7],
)
except CancelledError:
logger.error(
"Cancelled before attempting to reproduce experiment "
"'%s'",
rev[:7],
)
finally:
executor.cleanup()
return result
