.replace("ENDBITWIDTH",str(interval_end)) \
.replace("PLATFORM",options.platform_name)
os.system(("(cd PATH; cd PLATFORM; mkdir DIRNAME; cp STD_MICROBENCHPYNAME DIRNAME/MICROBENCHPYNAME)")\
.replace("PATH", pathHead) \
.replace("PLATFORM",options.platform_name) \
.replace("DIRNAME",dirName) \
.replace("STD_MICROBENCHPYNAME",std_microBenchPyName) \
.replace("MICROBENCHPYNAME",microBenchPyName))
scripts_ForMicroBenchmarks.append(pathHead+"/"+options.platform_name+"/"+dirName+"/"+microBenchPyName)
results_ForMicroBenchmarks.append(pathHead+"/"+options.platform_name+"/"+dirName+"/overview")
interval_head += intervalLen
processes_ForMicroBenchmarks = []
for script in scripts_ForMicroBenchmarks:
processes_ForMicroBenchmarks.append(Process(target=runMicroBenchmarks,args=(script,)))
for tmp_proc in processes_ForMicroBenchmarks:
tmp_proc.start()
for tmp_proc in processes_ForMicroBenchmarks:
tmp_proc.join()
resultDataStrs = []
for result_file_name in results_ForMicroBenchmarks:
subResultFile = open(result_file_name, 'r')
resultDataStrs = resultDataStrs + subResultFile.readlines()
subResultFile.close()
resultFile = open(pathHead+"/"+instName, 'w')
for data in resultDataStrs:
def get(url):
start = time.time()
try:
requests.get(url)
except:
ok = 0
else:
ok = 1
finally:
rt = time.time() - start
return {'ok': ok, 'rt': rt}
while True:
if task.empty():
print 'no task yet, wait 5s...'
time.sleep(5)
continue
try:
i = task.get(timeout=10)
'''
这里操作解析url, 查找商品，放入redis
'''
p = Process(target=deal_url, args=(format(i), ))
p.start()
p.join()
o = get(i)
#向resultQueue输出结果
result.put({'i': i, 'o': o})
except Exception, e:
print 'Error: {0}'.format(e)
import random
import time
from multiprocessing import Process
def get_random_string(length):
letters = string.ascii_lowercase
ret = ''.join(random.choice(letters) for i in range(length))
return ret
def worker(rep):
cli=LedgerCompliance.client.Client(API, HOST, PORT)
cli.connect()
for i in range(0,rep):
msg_key="KEY_"+get_random_string(8)
msg_val="VALUE_"+get_random_string(random.randint(50,1000))
cli.safeSet(msg_key.encode(),msg_val.encode())
res=cli.safeGet(msg_key.encode())
assert res.verified
proclist=[]
for i in range(0,4):
proclist.append(Process(target=worker, args=(100,)))
t0=time.time()
for p in proclist:
p.start()
for p in proclist:
p.join()
t1=time.time()
print("Run time: {} seconds".format(t1-t0))
self.channel = self.conn.channel()
self.channel.exchange_declare(exchange_name, 'topic')
queueId = self.channel.queue_declare( exclusive = True ).queue
self.channel.queue_bind(queueId, exchange_name, self.routingKey)
try:
self.channel.basic_consume(queueId, callback=self.callback_rdm)
except KeyboardInterrupt:
self.channel.close()
self.conn.close()
while True:
self.conn.drain_events()
def callback_rdm(self, message):
#print("Body:'%s', Proeprties:'%s', DeliveryInfo:'%s'" % ( message.body, message.properties, message.delivery_info))
print message.body
#message.ack()
#channel.basic_publish(message.body, exchange_name, arg_rky)
def close(self):
self.conn.close()
def processStart(rkey):
StateService(rkey).run()
if __name__ == '__main__':
s = r.grp['state']
for x in s:
p = Process(target=processStart, args=(x,)).start()
#p.join()
print(f'Result of {n} square : {result}')
# 메인
if __name__ == '__main__':
# 부모 프로세스 아이디
parent_process_id = os.getpid()
# 출력
print(f'Parent Process ID : {parent_process_id}')
# 프로세스 리스트 선언
processes = list()
# 프로세스 생성 및 실행
for i in range(1, 10):
# 생성
p = Process(name=str(i), target=square, args=(i, ))
# 배열에 담기
processes.append(p)
# 시작
p.start()
for process in processes:
process.join()
# 종료
print('Main-Processing Done!')
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('env_name', type=str)
parser.add_argument('--exp_name', type=str, default='vpg')
parser.add_argument('--render', action='store_true')
parser.add_argument('--discount', type=float, default=1.0)
parser.add_argument('--n_iter', '-n', type=int, default=100)
parser.add_argument('--batch_size', '-b', type=int, default=1000)
parser.add_argument('--ep_len', '-ep', type=float, default=-1.)
parser.add_argument('--learning_rate', '-lr', type=float, default=5e-3)
parser.add_argument('--reward_to_go', '-rtg', action='store_true')
parser.add_argument('--dont_normalize_advantages',
'-dna',
action='store_true')
parser.add_argument('--nn_baseline', '-bl', action='store_true')
parser.add_argument('--seed', type=int, default=1)
parser.add_argument('--n_experiments', '-e', type=int, default=1)
parser.add_argument('--n_layers', '-l', type=int, default=2)
parser.add_argument('--size', '-s', type=int, default=64)
args = parser.parse_args()
if not (os.path.exists('data')):
os.makedirs('data')
logdir = args.exp_name + '_' + args.env_name + \
'_' + time.strftime("%d-%m-%Y_%H-%M-%S")
logdir = os.path.join('data', logdir)
if not (os.path.exists(logdir)):
os.makedirs(logdir)
max_path_length = args.ep_len if args.ep_len > 0 else None
processes = []
for e in range(args.n_experiments):
seed = args.seed + 10 * e
print('Running experiment with seed %d' % seed)
def train_func():
train_PG(exp_name=args.exp_name,
env_name=args.env_name,
n_iter=args.n_iter,
gamma=args.discount,
min_timesteps_per_batch=args.batch_size,
max_path_length=max_path_length,
learning_rate=args.learning_rate,
reward_to_go=args.reward_to_go,
animate=args.render,
logdir=os.path.join(logdir, '%d' % seed),
normalize_advantages=not (args.dont_normalize_advantages),
nn_baseline=args.nn_baseline,
seed=seed,
n_layers=args.n_layers,
size=args.size)
# # Awkward hacky process runs, because Tensorflow does not like
# # repeatedly calling train_PG in the same thread.
p = Process(target=train_func, args=tuple())
p.start()
processes.append(p)
# if you comment in the line below, then the loop will block
# until this process finishes
# p.join()
for p in processes:
p.join()
def executedcommand(stack, rstack, lstack, com, opr, pc, pre, top, rtop, ltop,
address, value, tablecount, variable_region, lock,
process_number, process_path, count_pc, process_count,
terminate_flag, flag_number):
if com[pc] == 1: #push 演算スタックに即値を積む
top = push(opr[pc], stack, top)
pre = pc
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 2: #load 共有変数スタックから値を読み出し演算スタックに積む
value.acquire()
c = value[search_table(opr[pc], process_path)]
value.release()
top = push(c, stack, top)
pre = pc
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 3: #store 変数の値を更新し値スタックにそれまでの変数の値及びプロセス番号とパスを積む
value.acquire()
with open("value_stack.txt", 'a') as f:
f.write(
str(value[search_table(opr[pc], process_path)]) + ' ' +
str(process_number) + '.' + process_path + '\n')
f.close()
rtop.value = rtop.value + 2
(value[search_table(opr[pc], process_path)], top) = pop1(stack, top)
value.release()
pre = pc
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 4: #jpc スタックトップの値をポップしその値が1なら被演算子の番地にジャンプする
(c, top) = pop1(stack, top)
if c == 1:
pre = pc
pc = opr[pc] - 2
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 5: #jmp 無条件で被演算子の番地にジャンプする
pre = pc
pc = opr[pc] - 2
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 6: #op 被演算子の種類の演算を行う
if (opr[pc]) == 0: #'+'
(c, top) = pop1(stack, top)
(d, top) = pop1(stack, top)
top = push(c + d, stack, top)
elif (opr[pc]) == 1: #'*'
(c, top) = pop1(stack, top)
(d, top) = pop1(stack, top)
top = push(c * d, stack, top)
elif opr[pc] == 2: #'-'
(c, top) = pop1(stack, top)
(d, top) = pop1(stack, top)
top = push(d - c, stack, top)
elif opr[pc] == 3: #'>'
(c, top) = pop1(stack, top)
(d, top) = pop1(stack, top)
if d > c:
top = push(1, stack, top)
else:
top = push(0, stack, top)
elif opr[pc] == 4: #'=='
(c, top) = pop1(stack, top)
(d, top) = pop1(stack, top)
if d == c:
top = push(1, stack, top)
else:
top = push(0, stack, top)
pre = pc
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 7: #label ラベルスタックにジャンプ前のPCの値とプロセス番号を積む
if args[2] == 'f':
with open("label_stack.txt", 'a') as f:
f.write(str(pre + 1) + ' ' + str(process_number) + '\n')
f.close()
ltop.value = ltop.value + 2
pre = pc
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 21: #rjmp ラベルスタックから値を取り出しそのPCにジャンプする
a = count_pc - int(lstack[ltop.value])
ltop.value = ltop.value - 2
pre = pc
return (a, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 22: #restore 値スタックから値を取り出し共有変数スタックに保存する
s2 = re.search(r'([a-z]\d+\.)+', rstack[rtop.value + 1])
process_path = s2.group() + ".E"
value[search_table(opr[pc], process_path)] = int(rstack[rtop.value])
rtop.value = rtop.value - 2
pre = pc
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 19 or com[pc] == 28: #nop no operation
pre = pc
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 8 or com[pc] == 23: #par 並列ブロックの開始と終了を示す
#if opr[pc]==1:
# terminate_flag[flag_number]=1
# print(str(flag_number))
pre = pc
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 9: #alloc 新しい変数の番地を確保し初期値を0にする
#top=push(0,stack,top)
if args[2] == 'f':
value[tablecount.value] = 0
variable_region.append(0)
with open("variable_table.txt", 'a') as f:
f.write(str(opr[pc]) + '.' + process_path + ' 0\n')
tablecount.value = tablecount.value + 1
elif args[2] == 'b':
variable_path = search_table(opr[pc], process_path)
variable_region.append(0)
with open("variable_table.txt", 'r') as f:
variable_table = f.read().split('\n')
s = re.search(r'\s(-)?(\d+)', variable_table[variable_path])
variable_value = int(s.group().strip(' '))
value[search_table(opr[pc], process_path)] = variable_value
tablecount.value = tablecount.value + 1
pre = pc
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 10: #free 変数の番地の解放を示し値スタックにそれまでの値を保存する
table_address = search_table(opr[pc], process_path)
value.acquire()
with open("value_stack.txt", 'a') as f:
f.write(
str(value[search_table(opr[pc], process_path)]) + ' ' +
str(process_number) + '.' + process_path + '\n')
f.close()
value.release()
value[table_address] = 0
pre = pc
variable_region[opr[pc]] = value[opr[pc]]
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 11: #proc 手続きの開始 label命令とblock命令を行う 返り番地を演算スタックに積む
if args[2] == 'f':
with open("label_stack.txt", 'a') as f:
f.write(str(pre + 1) + ' ' + str(process_number) + '\n')
f.close()
push(pre + 1, stack, top)
pre = pc
process_path = 'p' + str(opr[pc]) + '.' + process_path
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 12: #ret 手続きの終了 演算スタックから返り番地を取り出しその番地にジャンプする
(c, top) = pop1(stack, top)
for i in range(0, len(process_path), 1):
if process_path[i] == '.':
process_path = process_path[i + 1:len(process_path)]
break
pre = pc
return (c, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 13: #block パスを追加
if com[pc + 3] == 16 and (com[pc + 1] == 5 or com[pc + 1] == 8):
process_path = 'c' + str(opr[pc]) + '.' + process_path
else:
process_path = 'b' + str(opr[pc]) + '.' + process_path
pre = pc
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 14: #end パスの削除
for i in range(0, len(process_path), 1):
if process_path[i] == '.':
process_path = process_path[i + 1:len(process_path)]
break
pre = pc
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 15: #fork 並列プロセスの生成
lock.release()
process = {}
start_process_count = process_count.value
already_terminate = {}
f = open('a' + (str)(opr[pc]) + '.txt', mode='r')
tables = f.read()
#並列ブロックを参照しそれぞれ開始番地と終了番地を取り出して各プロセスに与えプロセスを生成する
for i in range(0, len(tables), 10):
t1 = tables[i:i + 4]
s1 = re.search(r'\d+', t1)
t2 = tables[i + 5:i + 9]
s2 = re.search(r'\d+', t2)
terminate_flag[process_count.value] = 0
process[process_count.value] = Process(
target=execution,
args=(com, opr, (int)(s1.group()), (int)(s2.group()), count_pc,
stack, address, value, tablecount, rstack, lstack, rtop,
ltop, 0, variable_region, lock, process_number + '.' +
str(process_count.value - start_process_count + 1),
process_path, process_count, terminate_flag,
process_count.value))
process_count.value = process_count.value + 1
end_process_count = process_count.value
for i in range(start_process_count, process_count.value, 1):
process[i].start()
terminate_count = 0
#自分が生成したプロセスが終了しているかどうかを監視する．終了している場合terminateでプロセスを完全終了させる
for i in range(0, 100, 1):
already_terminate[i] = 0
while True:
for i in range(start_process_count, end_process_count, 1):
if terminate_flag[i] == 1 and already_terminate[i] == 0:
process[i].terminate()
process[i].join()
already_terminate[i] = 1
terminate_count = terminate_count + 1
if not process[i].is_alive():
process[i].join()
if terminate_count == end_process_count - start_process_count:
pre = pc
lock.acquire()
return (int(s2.group()), pre, stack, top, rtop, tablecount,
process_path)
pre = pc
lock.acquire()
return (a, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 16: #merge 並列ブロックの終了を示す
pre = pc
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 17: #func 関数の開始 label命令とblock命令を行い演算スタックトップの一つ下に帰り番地を積む
if args[2] == 'f':
with open("label_stack.txt", 'a') as f:
f.write(str(pre + 1) + ' ' + str(process_number) + '\n')
f.close()
(c, top) = pop1(stack, top)
push(pre + 1, stack, top)
push(c, stack, top)
pre = pc
process_path = 'f' + str(opr[pc]) + '.' + process_path
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 18: #f_return 関数の終了 演算スタックの一つしたの値を取り出しその番地にジャンプする
(d, top) = pop1(stack, top)
(c, top) = pop1(stack, top)
push(d, stack, top)
for i in range(0, len(process_path), 1):
if process_path[i] == '.':
process_path = process_path[i + 1:len(process_path)]
break
pre = pc
return (c, pre, stack, top, rtop, tablecount, process_path)
#elif com[pc]==19:#w_label whileの開始を示しlabel命令とblock命令を行う
#with open("label_stack.txt",'a') as f:
# f.write(str(pre+1)+' '+str(process_number)+'\n')
#f.close()
#process_path='w'+str(opr[pc])+'.'+process_path
#pre=pc
#return (pc+1,pre,stack,top,rtop,tablecount,process_path)
#elif com[pc]==20:#w_end whileの終了を示しlabel命令とend命令を行う
#with open("label_stack.txt",'a') as f:
# f.write(str(pre+1)+' '+str(process_number)+'\n')
#f.close()
#process_path=process_path.replace('w'+str(opr[pc])+'.','')
#return (pc+1,pre,stack,top,rtop,tablecount,process_path)
elif com[
pc] == 24: #r_alloc 逆向きのalloc命令 値スタックから値を取り出し 確保した共有変数スタックの番地に保存する
s2 = re.search(r'([a-z]\d+\.)+', rstack[rtop.value + 1])
process_path = s2.group()
with open("variable_table.txt", 'a') as f:
f.write(str(opr[pc]) + '.' + process_path + 'E 0\n')
value[tablecount.value] = int(rstack[rtop.value])
tablecount.value = tablecount.value + 1
rtop.value = rtop.value - 2
pre = pc
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 25: #r_free 逆向きのfree命令 変数の解放を示す
pre = pc
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 26: #r_fork 逆向きのfork命令 前向きとは逆順に並列テーブルを参照する
lock.release()
process = {}
start_process_count = process_count.value
f = open('a' + (str)(opr[pc]) + '.txt', mode='r')
already_terminate = {}
tables = f.read()
tables_process_number = int(len(tables) / 10)
print(str(tables_process_number))
for i in range(0, len(tables), 10):
t1 = tables[i:i + 4]
s1 = re.search(r'\d+', t1)
t2 = tables[i + 5:i + 9]
s2 = re.search(r'\d+', t2)
terminate_flag[process_count.value] = 0
process[process_count.value] = Process(
target=execution,
args=(com, opr, count_pc - (int)(s2.group()) + 1,
count_pc - (int)(s1.group()) + 1, count_pc, stack,
address, value, tablecount, rstack, lstack, rtop, ltop,
0, variable_region, lock, process_number + '.' +
str(process_count.value - start_process_count + 1),
process_path, process_count, terminate_flag,
process_count.value))
process_count.value = process_count.value + 1
end_process_count = process_count.value
for i in range(start_process_count, process_count.value, 1):
process[i].start()
terminate_count = 0
for i in range(0, 100, 1):
already_terminate[i] = 0
t3 = tables[0:0 + 4]
s3 = re.search(r'\d+', t3)
while True:
for i in range(start_process_count, end_process_count, 1):
if terminate_flag[i] == 1 and already_terminate[i] == 0:
process[i].terminate()
process[i].join()
already_terminate[i] = 1
terminate_count = terminate_count + 1
if not process[i].is_alive():
process[i].join()
if terminate_count == end_process_count - start_process_count:
pre = pc
lock.acquire()
return (count_pc - int(s3.group()) + 1, pre, stack, top, rtop,
tablecount, process_path)
for i in range(start_process_count, process_count.value, 1):
process[i].join()
a = count_pc - int(s3.group())
pre = pc
lock.acquire()
return (a, pre, stack, top, rtop, tablecount, process_path)
elif com[pc] == 27: #r_merge 逆向きのmerge
pre = pc
return (pc + 1, pre, stack, top, rtop, tablecount, process_path)
def compute_ml_data(filename, mode, previous = None):
print "Compute regression data: {}, {}, {}".format(filename, mode, previous)
df = pd.read_pickle(filename + "_post")
print "Loaded df {}".format(df.shape)
# ml_data = cPickle.load(open("ml_dataset"))
#truth_dict, truth_set = load_truth_set()
#positive_negative = cPickle.load(open("positive_negative.out"))
if mode == "train":
truth_dict, truth_set = load_truth_set()
positive_set = cPickle.load(open("blocking_positive_set_{}".format(mode)))
# Add all available training sets !
positive_set |= truth_set
positive_list = [((int(p[0]), int(p[1])), 1) for p in positive_set]
negative_set = cPickle.load(open("blocking_negative_set_{}".format(mode)))
negative_list = [((int(p[0]), int(p[1])), 0) for p in negative_set]
new_negatives = min(len(negative_list), 20 * len(positive_list))
print "Undersampling negatives: {} -> {}".format(len(negative_list), new_negatives)
negative_list = rnd.sample(negative_list, new_negatives)
assert not (negative_set & positive_set)
print "Train: {} {}".format(len(positive_list), len(negative_list))
positive_negative = positive_list + negative_list
del positive_set
del negative_set
del positive_list
del negative_list
del truth_set
del truth_dict
else:
total_set = cPickle.load(open("blocking_total_set_test"))
if previous:
prev_set = cPickle.load(open(previous[0]))
print "Previous blocking_set defined : {}".format(prev_set)
total_set = total_set - prev_set
total_list = [((int(p[0]), int(p[1])), 2) for p in total_set]
positive_negative = total_list
print "Test: {}".format(len(total_list))
#total_token_list = [t for el in df.norm_name for t in el.split() if len(t) >= 4]
#rarity_counter = Counter(total_token_list)
print "Garbage collection: {}".format(gc.collect())
rnd.shuffle(positive_negative)
#print positive_negative
#n_cores = cpu_count()
n_cores = 3
n_list = len(positive_negative)
n_block = n_list / n_cores + 1
jobs = []
data = []
for i in range(n_cores):
start_id = i * n_block
end_id = min (n_list, start_id + n_block)
jobs.append(Process(target = bulk_process, args = (df, positive_negative[start_id:end_id], "job_{}".format(i))))
for j in jobs:
j.start()
for j in jobs:
j.join()
print "Sleeping for 2 seconds"
time.sleep(2)
for i in range(n_cores):
print "Processing file {}".format(i)
tmp_filename = "bulk_process_{}_tmp".format("job_{}".format(i))
data_tmp = cPickle.load(open(tmp_filename, 'rb'))
system("rm -f {}".format(tmp_filename))
data.extend(data_tmp)
print "data length: {}".format(len(data))
ml_df = pd.DataFrame(data, columns = column_features())
if previous:
print "Previous df defined : {}".format(previous[1])
prev_df = pd.read_pickle(previous[1])
ml_df = pd.concat([ml_df, prev_df])
#print "Computing feature for {} pairs".format(len(positive_negative))
#data = Parallel(n_jobs=10)(delayed(compute_features)(df.loc[int(pair[0])], df.loc[int(pair[1])], index, pair, label) for index, (pair, label) in enumerate(positive_negative))
ml_df.to_pickle("ml_{}_set".format(mode))
def main(use_multiprocessing=False, log_level=None):
"""
Create and run a tello controller :
1) get the video stream from the tello
2) wait for keyboard commands to pilot the tello
3) optionnally, process the video frames to track a body and pilot the tello accordingly.
If use_multiprocessing is True, the parent process creates a child process ('worker')
and the workload is shared between the 2 processes.
The parent process job is to:
- get the video stream from the tello and displays it in an OpenCV window,
- write each frame in shared memory at destination of the child,
each frame replacing the previous one (more efficient than a pipe or a queue),
- read potential command from the child (currently, only one command:EXIT).
Commands are transmitted by a Pipe.
The child process is responsible of all the others tasks:
- process the frames read in shared memory (openpose, write_hud),
- if enable, do the tracking (calculate drone commands from position of body),
- read keyboard commands,
- transmit commands (from tracking or from keyboard) to the tello, and receive message from the tello.
"""
global tello
if use_multiprocessing:
# Create the pipe for the communication between the 2 processes
parent_cnx, child_cnx = Pipe()
else:
child_cnx = None
tello = TelloController(use_face_tracking=False,
kbd_layout="QWERTY",
write_log_data=False,
log_level=log_level, child_cnx=child_cnx)
first_frame = True
frame_skip = 300
for frame in tello.container.decode(video=0):
if 0 < frame_skip:
frame_skip = frame_skip - 1
continue
start_time = time.time()
if frame.time_base < 1.0/60:
time_base = 1.0/60
else:
time_base = frame.time_base
# Convert frame to cv2 image
frame = cv2.cvtColor(np.array(frame.to_image(),dtype=np.uint8), cv2.COLOR_RGB2BGR)
frame = cv2.resize(frame, (640,480))
if use_multiprocessing:
if first_frame:
# Create the shared memory to share the current frame decoded by the parent process
# and given to the child process for further processing (openpose, write_hud,...)
frame_as_ctypes = np.ctypeslib.as_ctypes(frame)
tello.shared_array = sharedctypes.RawArray(frame_as_ctypes._type_, frame_as_ctypes)
tello.frame_shape = frame.shape
first_frame = False
# Launch process child
p_worker = Process(target=openpose_worker)
p_worker.start()
# Write the current frame in shared memory
tello.shared_array[:] = np.ctypeslib.as_ctypes(frame.copy())
# Check if there is some message from the child
if parent_cnx.poll():
msg = parent_cnx.recv()
if msg == "EXIT":
print("MAIN EXIT")
p_worker.join()
tello.drone.quit()
cv2.destroyAllWindows()
exit(0)
else:
frame = tello.process_frame(frame)
tello.sound_player.play()
if not use_multiprocessing: tello.fps.update()
# Display the frame
cv2.imshow('Tello', frame)
cv2.waitKey(1)
frame_skip = int((time.time() - start_time)/time_base)
def test_read(self):
##
## run service
##
checker_service_process = Process(target=vote_contract.run_checker_service)
checker_service_process.start()
time.sleep(0.1)
##
## create transaction
##
# number of voters and values
options = ['alice', 'bob', 'sally']
num_voters = 50
values = [[1, 0, 0] for _ in range(0, num_voters)]
# create keys and particpants
params = setup()
(tally_priv, tally_pub) = key_gen(params)
keys = [key_gen(params) for _ in range(0, num_voters)]
participants = [pack(pub) for (_, pub) in keys]
# get init token
init_transaction = vote.init()
# get initial scores
create_vote_transaction = vote.create_vote(
(init_transaction['transaction']['outputs'][0],),
None,
None,
dumps(options),
dumps(participants),
pack(tally_priv),
pack(tally_pub)
)
vote_0 = create_vote_transaction['transaction']['outputs'][1]
# add votes
transaction = {}
input_obj = vote_0
for i in range(0, num_voters):
transaction = vote.add_vote(
(input_obj,),
None,
None,
dumps(values[i]), # votes' valu (0 or 1)
pack(keys[i][0]), # voter's priv key
pack(keys[i][1]) # voter's pub key
)
input_obj = transaction['transaction']['outputs'][0]
# tally
transaction = vote.tally(
(input_obj,),
None,
None,
pack(tally_priv),
pack(tally_pub)
)
result = transaction['transaction']['outputs'][0]
# read result
transaction = vote.read(
None,
(result,),
None,
)
# print result
print transaction['transaction']['returns']
##
## submit transaction
##
response = requests.post(
'http://127.0.0.1:5000/' + vote_contract.contract_name + '/read', json=transaction_to_solution(transaction)
)
self.assertTrue(response.json()['success'])
##
## stop service
##
checker_service_process.terminate()
checker_service_process.join()
def process_multi_threaded(fasta,
bam_fname,
fastq1,
sidecar_fname,
fastq2=None,
threads=1,
max_templates=None,
platform='Illumina',
sample_name='Seven',
cigar_v2=True,
do_not_index=False):
"""
:param bam_fname:
:param bam_hdr:
:param fastq1:
:param sidecar_fname: File containing just the long qnames
:param fastq2:
:param threads:
:param max_templates:
:param platform:
:param sample_name:
:param cigar_v2: If True, write out CIGARs in V2 format
:param do_not_index: If True, the output BAMs will be collated into one bam, sorted and indexed
the N output BAMs created by the individual workers will be deleted at the end.
If False, the N output BAMs created by the individual workers will remain. This
option allows users to merge, sort and index the BAM fragments with their own tools
:return:
Note: The pysam sort invocation expects 1GB/thread to be available
"""
long_qname_table = load_qname_sidecar(sidecar_fname)
rg_id = 'rg{}'.format(hash(' '.join(sys.argv)))
fasta_ann = fasta + '.ann'
bam_hdr = construct_header(fasta_ann,
rg_id=rg_id,
sample=sample_name,
platform=platform)
# Start worker processes
logger.debug('Starting {} processes'.format(threads))
file_fragments = [
'{}.{:04}.bam'.format(bam_fname, i) for i in range(threads)
]
in_queue = Queue()
p_list = [
Process(target=disciple,
args=(file_fragments[i], bam_hdr, rg_id, long_qname_table,
cigar_v2, in_queue)) for i in range(threads)
]
for p in p_list:
p.start()
t0 = time.time()
# Burn through file
logger.debug('Starting to read FASTQ file')
fastq_l = [pysam.FastxFile(fastq1)]
if fastq2 is not None: fastq_l += [pysam.FastxFile(fastq2)]
cnt = 0
for cnt, reads in enumerate(zip(*fastq_l)):
# qname, [(seq, qual) ... ]
in_queue.put((reads[0].name, [(r.sequence, r.quality) for r in reads]))
if max_templates is not None and cnt >= max_templates:
break
if cnt % 100000 == 0:
logger.debug('Read {} templates'.format(cnt))
# Tell child processes to stop
logger.debug('Stopping child processes')
for i in range(threads):
in_queue.put(__process_stop_code__)
# Wait for them to finish
for p in p_list:
p.join()
t1 = time.time()
logger.debug('Processed {} templates in {:0.2f}s ({:0.2f} t/s)'.format(
cnt, t1 - t0, cnt / (t1 - t0)))
merge_sorted_fragments(bam_fname,
file_fragments,
do_not_index=do_not_index)
def test_add_vote(self):
##
## run service
##
checker_service_process = Process(target=vote_contract.run_checker_service)
checker_service_process.start()
time.sleep(0.1)
##
## create transaction
##
# create keys
params = setup()
(tally_priv, tally_pub) = key_gen(params)
(voter1_priv, voter1_pub) = key_gen(params)
(_, voter2_pub) = key_gen(params)
(_, voter3_pub) = key_gen(params)
# set up options, particpants, and tally's key
options = ['alice', 'bob']
participants = [pack(voter1_pub), pack(voter2_pub), pack(voter3_pub)]
# get init token
init_transaction = vote.init()
token = init_transaction['transaction']['outputs'][0]
# initialise vote (all votes are zero)
create_vote_transaction = vote.create_vote(
(token,),
None,
None,
dumps(options),
dumps(participants),
pack(tally_priv),
pack(tally_pub)
)
old_vote = create_vote_transaction['transaction']['outputs'][1]
# add a vote
transaction = vote.add_vote(
(old_vote,),
None,
None,
dumps([1, 0]),
pack(voter1_priv),
pack(voter1_pub)
)
print transaction
##
## submit transaction
##
response = requests.post(
'http://127.0.0.1:5000/' + vote_contract.contract_name + '/add_vote', json=transaction_to_solution(transaction)
)
self.assertTrue(response.json()['success'])
##
## stop service
##
checker_service_process.terminate()
checker_service_process.join()
def main(_):
parser = argparse.ArgumentParser(description='TransE.')
parser.add_argument('--lr', dest='lr', type=float, help="Learning rate", default=0.005)
parser.add_argument('--L1_flag', dest='L1_flag', type=int, help="norm method", default=0)
parser.add_argument('--margin', dest='margin', type=int, help="margin", default=1)
parser.add_argument('--data', dest='data_dir', type=str, help="Data folder", default='../../data/baike/')
parser.add_argument("--dim", dest='dim', type=int, help="Embedding dimension", default=150)
parser.add_argument("--worker", dest='n_worker', type=int, help="Evaluation worker", default=3)
parser.add_argument("--load_model", dest='load_model', type=str, help="Model file:xxx.meta", default=None)
parser.add_argument("--max_iter", dest='max_iter', type=int, help="Max iteration", default=500)
parser.add_argument("--train_batch", dest="train_batch", type=int, help="Training batch size", default=10240)
parser.add_argument("--eval_batch", dest="eval_batch", type=int, help="Evaluation batch size", default=40960)
parser.add_argument("--optimizer", dest='optimizer', type=str, help="Optimizer", default='gradient')
parser.add_argument("--generator", dest='n_generator', type=int, help="Data generator", default=10)
parser.add_argument("--save_dir", dest='save_dir', type=str, help="Model path", default='./log/')
parser.add_argument("--save_per", dest='save_per', type=int, help="Save per x iteration", default=50)
parser.add_argument("--eval_per", dest='eval_per', type=int, help="Evaluate every x iteration", default=5)
parser.add_argument("--summary_dir", dest='summary_dir', type=str, help="summary directory",default='./TransE_summary/')
parser.add_argument("--keep", dest='drop_out', type=float, help="Keep prob (1.0 keep all, 0. drop all)",default=0.5)
parser.add_argument("--pad", dest='candidate_dim', type=int, help="dimension of the candidate",default=55)
parser.add_argument("--prefix", dest='prefix', type=str, help="model_prefix", default='neg 1 valid')
args=parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"]="5"
model=TransE(data_dir=args.data_dir,train_batch=args.train_batch,eval_batch=args.eval_batch,L1_flag=args.L1_flag,margin=args.margin, dim = args.dim)
pos_triple,neg_triple,train_loss,train_op = train_ops(model,learning_rate=args.lr,optimizer_str=args.optimizer)
test_input,test_head,test_tail,test_relation=test_ops(model)
normalize_entity_op,normalize_relation_op=normalize_ops(model)
candidate_test_input,candidate_input,candidate_tail_ids, candidate_hrt_res = pre_in_candidates_ops(model)
init=tf.global_variables_initializer()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
saver = tf.train.Saver()
# run tensorflow
with tf.Session(config=config) as sess:
sess.run(init)
iter_offset = 0
total_inst=model.n_train
raw_training_data_queue = Queue()
training_data_queue = Queue()
evaluation_queue=JoinableQueue()
result_queue=Queue()
data_generators = list()
for i in range(args.n_generator):
data_generators.append(Process(target=data_generator_func,args=(
raw_training_data_queue,training_data_queue,model.right_num,model.left_num,model.train_tr_h,model.train_hr_t,model.train_ht_r,
model.n_entity,model.n_relation)))
data_generators[-1].start()
for i in range(args.n_worker):
worker=Process(target=worker_func,args=(evaluation_queue,result_queue,
model.tr_h,model.hr_t,model.ht_r))
worker.start()
print("work %d start!"% i)
for n_iter in range(iter_offset,args.max_iter):
start_time = timeit.default_timer()
accu_loss = 0.0
ninst = 0
nbatches_count = 0
for dat in model.raw_training_data(batch_size=args.train_batch):
raw_training_data_queue.put(dat)
nbatches_count += 1
while nbatches_count > 0:
nbatches_count -= 1
pos_triple_batch,neg_triple_batch= training_data_queue.get()
sess.run([normalize_entity_op,normalize_relation_op])
loss, _= sess.run([train_loss,train_op], feed_dict={pos_triple:pos_triple_batch,
neg_triple:neg_triple_batch})
accu_loss += loss
ninst += pos_triple_batch.shape[0]
if ninst % (5000) is not None:
print(
'[%d sec](%d/%d) : %.2f -- loss : %.5f ' % (
timeit.default_timer() - start_time, ninst, total_inst, float(ninst) / total_inst,
loss / (pos_triple_batch.shape[0])),end='\r')
print("iter %d avg loss %.5f, time %.3f" % (n_iter, accu_loss / ninst, timeit.default_timer() - start_time))
if n_iter != 0 and (n_iter % args.save_per == 0 or n_iter == args.max_iter - 1):
save_path=saver.save(sess,os.path.join(
args.save_dir,"TransE_"+str(args.prefix)+"_"+str(n_iter)+".ckpt"))
print("TransE Model saved at %s" % save_path)
if n_iter!=0 and (n_iter % args.eval_per == 0 or n_iter == args.max_iter - 1):
test_start_time=timeit.default_timer()
accu_mean_rank_t = list()
hit_dis = []
na_dis = []
evaluation_count = 0
candidate_len = [0 for i in range(56)]
hit = [0.0 for i in range(56)]
for (testing_data,candidates) in model.candidate_data(batch_size=args.eval_batch):
candidate_tail_pred, candidate_tail_res =sess.run([candidate_tail_ids, candidate_hrt_res],{candidate_test_input: testing_data,candidate_input:candidates})
temp_candidate_len , temp_hit, temp_hit_dis, temp_na_dis = candidate_evaluation_distance(testing_data,candidates, candidate_tail_pred, candidate_tail_res)
candidate_len += temp_candidate_len
hit += temp_hit
hit_dis += temp_hit_dis
na_dis += temp_na_dis
evaluation_queue.put((testing_data,candidates,candidate_tail_pred))
evaluation_count += 1
print(hit_dis)
np.save("./summary/hit_dis.npy", hit_dis)
np.save("./summary/na_dis.npy", na_dis)
candidate_len_list = []
acc_list = []
for i in range(56):
if candidate_len[i] != 0:
print("|",i,"|", candidate_len[i],"|", round(candidate_len[i]/408867.0,3), "|", hit[i], "|",round(hit[i]/candidate_len[i],3),"|")
candidate_len_list.append(i)
acc_list.append(round(hit[i]/candidate_len[i],3))
print("hit_dis: ", np.mean(np.asarray(hit_dis)))
print("na_dis: ", np.mean(np.asarray(na_dis)))
with open("./summary/candidate_len_list.json", "w") as f:
json.dump(candidate_len_list, f)
with open("./summary/acc_list_neg1_l2_valid.json", "w") as f:
json.dump(acc_list, f)
for i in range(args.n_worker):
evaluation_queue.put(None)
print("waiting for worker finishes their work")
evaluation_queue.join()
print("all worker stopped.")
while evaluation_count > 0:
evaluation_count -= 1
mrt = result_queue.get()
accu_mean_rank_t += mrt
print('cost time:[%.3f sec]'%(timeit.default_timer()-test_start_time))
print("margin: ", args.margin, "dim : ", args.dim, "batch_size", args.train_batch)
print(
"INITIALIZATION [TAIL PREDICTION] MEAN RANK: %.1f HIT@10 %.3f HIT@1 %.3f tail not in candidate %.3f" %
(np.mean(accu_mean_rank_t),
np.mean(np.asarray(accu_mean_rank_t, dtype=np.int32) < 10),
np.mean(np.asarray(accu_mean_rank_t, dtype=np.int32) < 1),
np.mean(np.asarray(accu_mean_rank_t, dtype=np.int32) == 55)))
for i in range(args.n_generator):
data_generators[i].terminate()
from multiprocessing import Semaphore,Process
from time import sleep
import os
# 创建信号量
sem = Semaphore(3)
# 系统中最多有3个进程同时执行该事件
def fun():
sem.acquire() # 消耗信号量
print("%d执行事件"%os.getpid())
sleep(3)
print("%d执行完毕"%os.getpid())
sem.release() # 增加信号量
jobs = []
for i in range(5):
p = Process(target = fun)
jobs.append(p)
p.start()
for i in jobs:
i.join()
print(sem.get_value())
# t3.start()
# t4.start()
# t5.start()
# t6.start()
# t7.start()
# t8.start()
# t1.join()
# t2.join()
# t3.join()
# t4.join()
# t5.join()
# t6.join()
# t7.join()
# t8.join()
p1 = Process(target=batch1, args=(100000000, 102000000))
p2 = Process(target=batch1, args=(102000001, 104000001))
p3 = Process(target=batch1, args=(104000002, 106000002))
p4 = Process(target=batch1, args=(106000003, 108000003))
p5 = Process(target=batch1, args=(108000004, 110000004))
p6 = Process(target=batch1, args=(110000005, 112000005))
p7 = Process(target=batch1, args=(112000006, 114000006))
p8 = Process(target=batch1, args=(114000007, 116000007))
p1.start()
p2.start()
p3.start()
p4.start()
p5.start()
p6.start()
p7.start()
p8.start()
def p_start(text, speed):
p = Process(target=loading, args=(text, speed,))
p.start()
return p
from multiprocessing import Process, Queue #这里的Queue是multiprocessing里面的自带queue模块，跟Queue模块有区别，应该会有继承关系
import Queue as Q2
def f(q, n):
q.put([n, 'hello']) #注意get是随机拿一条 所以这个例子是进程间共享数据
print q.get()
print q.get()
if __name__ == '__main__':
#q = Queue()
q = Q2.Queue()
q.put('ddd')
for i in range(5):
p = Process(target=f, args=(q, i))
p.start()
while True:
print q.get()
#for i in range(5):
# print i #linux 下运行
'''
#!/user/bin/python
#_*_coding:utf-8 _*_
from multiprocessing import Process,Queue #这里的Queue是multiprocessing里面的自带queue模块，跟Queue模块有区别，应该会有继承关系
#import Queue as Q2
def f(q,n):
q.put([n,'hello']) #注意put是随机拿一条 所以这个例子是进程间共享数据
print q.get()
print q.get()
if 'hashmeter' not in settings: settings['hashmeter'] = 0 if 'scantime' not in settings: settings['scantime'] = 30L if 'rpcuser' not in settings or 'rpcpass' not in settings: print "Missing username and/or password in cfg file" sys.exit(1) settings['port'] = int(settings['port']) settings['threads'] = int(settings['threads']) settings['hashmeter'] = int(settings['hashmeter']) settings['scantime'] = long(settings['scantime']) thr_list = [] for thr_id in range(settings['threads']): p = Process(target=miner_thread, args=(thr_id,)) p.start() thr_list.append(p) time.sleep(1) # stagger threads print settings['threads'], "mining threads started" print time.asctime(), "Miner Starts - %s:%s" % (settings['host'], settings['port']) try: for thr_proc in thr_list: thr_proc.join() except KeyboardInterrupt: pass print time.asctime(), "Miner Stops - %s:%s" % (settings['host'], settings['port'])
def start_receive(self, channel, end_word='end'):
if self._job:
return
self._job = Process(target=self._receive_core,
args=(channel, end_word))
self._job.start()
def main():
# Get ip address of the given interface
ip_address = None
try:
ip_address = get_ip_address(sys.argv[1])
except Exception as e:
print("Something wrong with arguments", e)
exit(-1)
print("Starting the video server on {}".format(ip_address))
# Define Video Capture object and auxilary objects
cap = VideoCapture()
shape = cap.get_size()
stream = cap.get_stream_function()
# Define shared values
shared_array_base = Array(ctypes.c_uint8, shape[0] * shape[1] * shape[2])
frame = np.ctypeslib.as_array(shared_array_base.get_obj())
frame = frame.reshape(shape[0], shape[1], shape[2])
finished = Value('i', 0)
write_video_flag = Value('i', 0)
shared_fps = Value('f', 0)
# Define processes
video_process = Process(target=stream, args=(frame, finished, shared_fps))
motion_detector_process = Process(target=motion_detector,
args=(frame, finished, write_video_flag,
shape))
writer_process = Process(target=writer,
args=(frame, finished, write_video_flag, shape))
# Launch capture process
video_process.start()
# Sleep for some time to allow videocapture start working first
time.sleep(5)
# Launch the rest processes
motion_detector_process.start()
writer_process.start()
global server
def terminate():
cap.release()
finished.value = True
video_process.terminate()
motion_detector_process.terminate()
writer_process.terminate()
server.shutdown()
try:
# Start the server on the ip address
camhandler = get_cam_handler(frame, finished, shared_fps, ip_address)
server = ThreadedHTTPServer((ip_address, 8080), camhandler)
print("Server started")
server.serve_forever()
except KeyboardInterrupt:
# Release everything
terminate()
exit(0)
except Exception as e:
# Something wrong happened
print("Exception", e)
terminate()
exit(-1)
from multiprocessing import Process, Manager
def f(d):
d[1] += '1'
d['2'] += 2
if __name__ == '__main__':
manager = Manager()
d = manager.dict()
d[1] = '1'
d['2'] = 2
p1 = Process(target=f, args=(d,))
p2 = Process(target=f, args=(d,))
p1.start()
p2.start()
p1.join()
p2.join()
print d
def manage_reconstruction(proc, experiment_dir, rec_id=None):
"""
This function starts the interruption discovery process and continues the recontruction processing.
It reads configuration file defined as <experiment_dir>/conf/config_rec.
If multiple generations are configured, or separate scans are discovered, it will start concurrent reconstructions.
It creates image.npy file for each successful reconstruction.
Parameters
----------
proc : str
processing library, choices are: cpu, cuda, opencl
experiment_dir : str
directory where the experiment files are loacted
rec_id : str
optional, if given, alternate configuration file will be used for reconstruction, (i.e. <rec_id>_config_rec)
Returns
-------
nothing
"""
if os.path.exists('stopfile'):
os.remove('stopfile')
print('starting reconstruction')
# the rec_id is a postfix added to config_rec configuration file. If defined, use this configuration.
conf_dir = os.path.join(experiment_dir, 'conf')
if rec_id is None:
conf_file = os.path.join(conf_dir, 'config_rec')
else:
conf_file = os.path.join(conf_dir, rec_id + '_config_rec')
# check if file exists
if not os.path.isfile(conf_file):
print('no configuration file ' + conf_file + ' found')
return
# verify the configuration file
if not ver.ver_config_rec(conf_file):
# if not verified, the ver will print message
return
try:
config_map = ut.read_config(conf_file)
if config_map is None:
print("can't read configuration file " + conf_file)
return
except Exception as e:
print('Cannot parse configuration file ' + conf_file +
' , check for matching parenthesis and quotations')
print(str(e))
return
# exp_dirs_data list hold pairs of data and directory, where the directory is the root of data/data.tif file, and
# data is the data.tif file in this directory.
exp_dirs_data = []
# experiment may be multi-scan in which case reconstruction will run for each scan
for dir in os.listdir(experiment_dir):
if dir.startswith('scan'):
datafile = os.path.join(experiment_dir, dir, 'data', 'data.tif')
if os.path.isfile(datafile):
exp_dirs_data.append(
(datafile, os.path.join(experiment_dir, dir)))
# if there are no scan directories, assume it is combined scans experiment
if len(exp_dirs_data) == 0:
# in typical scenario data_dir is not configured, and it is defaulted to <experiment_dir>/data
# the data_dir is ignored in multi-scan scenario
try:
data_dir = config_map.data_dir
except AttributeError:
data_dir = os.path.join(experiment_dir, 'data')
if os.path.isfile(os.path.join(data_dir, 'data.tif')):
exp_dirs_data.append((os.path.join(data_dir,
'data.tif'), experiment_dir))
elif os.path.isfile(os.path.join(data_dir, 'data.npy')):
exp_dirs_data.append((os.path.join(data_dir,
'data.npy'), experiment_dir))
no_runs = len(exp_dirs_data)
if no_runs == 0:
print('did not find data.tif nor data.npy file(s). ')
return
try:
generations = config_map.generations
except:
generations = 0
try:
reconstructions = config_map.reconstructions
except:
reconstructions = 1
device_use = []
if proc == 'cpu':
cpu_use = [-1] * reconstructions
if no_runs > 1:
for _ in range(no_runs):
device_use.append(cpu_use)
else:
device_use = cpu_use
else:
try:
devices = config_map.device
except:
devices = [-1]
if no_runs * reconstructions > 1:
if exp_dirs_data[0][0].endswith('tif'):
data_shape = ut.read_tif(exp_dirs_data[0][0]).shape
elif exp_dirs_data[0][0].endswith('npy'):
data_shape = np.load(exp_dirs_data[0][0]).shape
device_use = get_gpu_use(devices, no_runs, reconstructions,
data_shape)
else:
device_use = devices
# start the interrupt process
interrupt_process = Process(target=interrupt_thread, args=())
interrupt_process.start()
if no_runs == 1:
if len(device_use) == 0:
device_use = [-1]
dir_data = exp_dirs_data[0]
datafile = dir_data[0]
dir = dir_data[1]
if generations > 1:
gen_rec.reconstruction(proc, conf_file, datafile, dir, device_use)
elif reconstructions > 1:
mult_rec.reconstruction(proc, conf_file, datafile, dir, device_use)
else:
rec.reconstruction(proc, conf_file, datafile, dir, device_use)
else:
if len(device_use) == 0:
device_use = [[-1]]
else:
# check if is it worth to use last chunk
if proc != 'cpu' and len(device_use[0]) > len(device_use[-1]) * 2:
device_use = device_use[0:-1]
if generations > 1:
r = 'g'
elif reconstructions > 1:
r = 'm'
else:
r = 's'
q = Queue()
for gpus in device_use:
q.put((None, gpus))
# index keeps track of the multiple directories
index = 0
processes = {}
pr = []
while index < no_runs:
pid, gpus = q.get()
if pid is not None:
os.kill(pid, signal.SIGKILL)
del processes[pid]
datafile = exp_dirs_data[index][0]
dir = exp_dirs_data[index][1]
p = Process(target=rec_process,
args=(proc, conf_file, datafile, dir, gpus, r, q))
p.start()
pr.append(p)
processes[p.pid] = index
index += 1
for p in pr:
p.join()
# close the queue
q.close()
interrupt_process.terminate()
print('finished reconstruction')
def main(model,
saveto,
k=5,
normalize=False,
zero_pad=False,
n_process=5,
datasets='dev,test',
sampling=False,
pkl_name=None):
# load model model_options
if pkl_name is None:
pkl_name = model
with open('%s.pkl' % pkl_name, 'rb') as f:
options = pkl.load(f)
# fetch data, skip ones we aren't using to save time
load_data, prepare_data = get_dataset(options['dataset'])
_, valid, test, worddict = load_data(
load_train=False,
load_dev=True if 'dev' in datasets else False,
load_test=True if 'test' in datasets else False)
# <eos> means end of sequence (aka periods), UNK means unknown
word_idict = dict()
for kk, vv in worddict.iteritems():
word_idict[vv] = kk
word_idict[0] = '<eos>'
word_idict[1] = 'UNK'
# create processes
queue = Queue()
rqueue = Queue()
processes = [None] * n_process
for midx in xrange(n_process):
processes[midx] = Process(target=gen_model,
args=(queue, rqueue, midx, model, options, k,
normalize, word_idict, sampling))
processes[midx].start()
# index -> words
def _seqs2words(caps):
capsw = []
for cc in caps:
ww = []
for w in cc:
if w == 0:
break
ww.append(word_idict[w])
capsw.append(' '.join(ww))
return capsw
# unsparsify, reshape, and queue
def _send_jobs(contexts):
for idx, ctx in enumerate(contexts):
cc = ctx.todense().reshape([14 * 14, 512])
if zero_pad:
cc0 = numpy.zeros(
(cc.shape[0] + 1, cc.shape[1])).astype('float32')
cc0[:-1, :] = cc
else:
cc0 = cc
queue.put((idx, cc0))
# retrieve caption from process
def _retrieve_jobs(n_samples):
caps = [None] * n_samples
for idx in xrange(n_samples):
resp = rqueue.get()
caps[resp[0]] = resp[1]
if numpy.mod(idx, 10) == 0:
print 'Sample ', (idx + 1), '/', n_samples, ' Done'
return caps
ds = datasets.strip().split(',')
# send all the features for the various datasets
for dd in ds:
if dd == 'dev':
print 'Development Set...',
_send_jobs(valid[1])
caps = _seqs2words(_retrieve_jobs(len(valid[1])))
with open(saveto + '.dev.txt', 'w') as f:
print >> f, '\n'.join(caps)
print 'Done'
if dd == 'test':
print 'Test Set...',
_send_jobs(test[1])
caps = _seqs2words(_retrieve_jobs(len(test[1])))
with open(saveto + '.test.txt', 'w') as f:
print >> f, '\n'.join(caps)
print 'Done'
# end processes
for midx in xrange(n_process):
queue.put(None)
print('task1: hello, %s' % msg)
time.sleep(1)
def task2(msg):
print('task2: hello, %s' % msg)
time.sleep(1)
def task3(msg):
print( 'task3: hello, %s' % msg)
time.sleep(1)
if __name__ == '__main__':
p1 = Process(target=task1, args=('one',))
p2 = Process(target=task2, args=('two',))
p3 = Process(target=task3, args=('three',))
start = time.time()
p1.start()
p2.start()
p3.start()
print("The number of CPU is:" + str(multiprocessing.cpu_count()))
for p in multiprocessing.active_children():
print("child p.name: " + p.name + "\tp.id: " + str(p.pid))
p1.join()
p2.join()
def run(host='127.0.0.1', port=5556):
Process(target=_serve, args=(host, port)).start()
def build_hash_breaker_process(word_generator, batch_size, start):
return Process(target=find_hash_original,
args=(hashlib.md5,
{next(word_generator)
for _ in range(batch_size)
}, read_shadow('shadow'), start))
def run(self):
print('IP processing running')
valid_process = Process(target=Schedule.valid_proxy)
check_process = Process(target=Schedule.check_pool)
valid_process.start()
check_process.start()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print(get_local_ip())
s.bind((str(get_local_ip()), 5555))
s.listen()
conn, addr = s.accept()
print("Connection address: " + str(addr))
while True:
data = conn.recv(20)
if not data:
break
server_ip = str(data.decode())
print("Server IP is: " + server_ip)
P1 = Process(target=run_OSC, args=[motor_values])
P1.start()
fig, ax = plt.subplots()
p1 = plt.bar(0, initial_motor_state, color='b')
ax.set_ylim(0, 100)
anim = animation.FuncAnimation(fig,
animate,
interval=0,
frames=None,
fargs=[p1],
repeat=False,
blit=True)
plt.show()
]
print(row)
else:
row = [
smart2, smart3, smart4, smart5, smart9, smart193,
smart194, smart195, smart196, smart197, smart198,
0, 100
]
writer.writerow(row)
if __name__ == '__main__':
file_list = []
os_csv_path("/ssd/diskcsv", file_list)
fileListALL = create_fileprocess(file_list, processes_num)
processes = list()
#write_csv(fileListALL[0], 'f**k.csv')
for i in range(processes_num):
#p = Process(target=csv_ana, args=(fileListALL[i], f_l, sige_all[i],i,))
input_file = "12thread" + str(i) + ".csv"
p = Process(target=write_csv, args=(
fileListALL[i],
input_file,
))
p.start()
processes.append(p)
for p in processes:
p.join()
MyManager.register('Ppo', PPO)
manager = MyManager()
manager.start()
Ppo = manager.Ppo(action_dim, observation_dim, is_train=True)
# GLOBAL_PPO = PPO(action_dim, observation_dim, is_train=True) #一个global的ppo
print('<TRAIN_LOG> ', 'GLOBAL_PPO get', ' time:', datetime.now())
try:
process = []
# 为每个worker创建进程
workers = [Worker(wid=i, ppo=Ppo) for i in range(N_WORKER)]
for i, worker in enumerate(workers):
p = Process(target=worker.work, ) # 创建进程
p.start() # 开始进程
print('<TRAIN_LOG> ', ' worker process: ', i, 'pid:', p.pid, ' time:', datetime.now())
process.append(p) # 把进程放进进程列表里，方便管理
# 把一个全局的PPO更新放到进程列表最后
p = Process(target=Ppo.update, )
p.start()
print('update process', 'pid', p.pid, ' time:', datetime.now())
p.join()
process.append(p)
for p in process:
p.join()
except Exception as e:
print("exception occurred")
