def main():
# 0.获取原来要copy的文件夹的名字
oldFolderName = input('请输入文件夹的名字：')
# 1.创建一个文件夹
newFolderNmae = oldFolderName + '复件'
os.mkdir(newFolderNmae)
# 2.获取old文件夹中的所有的文件夹名
fileNames = os.listdir(oldFolderName)
# 3.使用多进程的方式copy，原文件中所有文件到新的文件夹中
pool = Pool(5)
queue = Manager().queue()
# 不能把列表给子进程，如果给列表的话，相当于不是多任务执行，而是一个进程把所有的列表都在复制，所以这个用循环，每个文件给一个子进程
for name in fileNames:
pool.apply_async(copyFileTask, args=(name, oldFolderName, newFolderNmae, queue))
num = 0
allNum = len(fileNames)
while True:
queue.get()
num += 1
copyRate = num / allNum
print('\rcopy的进度是：%.2f%%' % (copyRate * 100), end='')
if num == allNum:
break
def main():
# 0.获取用户要拷贝的文件夹的名字
oldFolderName = input("请输入要拷贝的文件夹名：")
# 1.创建一个文件夹
newFolderName = oldFolderName + "-复件"
os.mkdir(newFolderName)
# 2.获取源文件夹中的所有文件名
fileNames = os.listdir(oldFolderName)
# 3.使用多进程的方式拷贝源文件夹中的所有文件到新的文件夹中
pool = Pool(5)
queue = Manager().Queue()
for name in fileNames:
pool.apply_async(copyFileTask,
args=(name, oldFolderName, newFolderName, queue))
num = 0
allNum = len(fileNames)
while True:
queue.get()
num += 1
copyRate = num / allNum
print("\r复制文件的进度是：%.2f%%" % (copyRate * 100), end="")
if num == allNum:
print("")
break
print("%s文件夹拷贝完成" % oldFolderName)
def main():
#1.获得用户输入的文件夹名字
oldFileName = input("请输入需要copy的文件夹名字：")
#2.创建一个文件夹
newFileName = oldFileName+"[复件]"
os.mkdir(newFileName)
#3.获得old文件中所以的文件名
fileNames = os.listdir(oldFileName)
#rint(fileNames)
#4.使用进程池copy文件内容
pool = Pool(5)
queue = Manager().Queue()
for names in fileNames:
pool.apply_async(copyFile,(names,oldFileName,newFileName,queue))
num = 0
allNum = len(fileNames)
while num < allNum:
queue.get()
num +=1
cn =num/allNum
print("\r\rcopy进度为:%.2f%%" % (cn * 100), end="")
pool.close()
pool.join()
print("")
print("copy完成！")
def main():
# 1.首先获取旧的文件夹名字
oldFolder = input("请输入您想要复制的文件夹名字:")
# 2.赋予新文件夹新的名字
newFolder = oldFolder + "-副件"
os.mkdir(newFolder) #新建文件夹
fileName = os.listdir(oldFolder) #获取文件夹中的所有文件
p = Pool(5) #创建进程池
queue = Manager().Queue() #进程内部数据共享
allNum = len(fileName)
num = 0
for name in fileName:
p.apply_async(copy, args=(name, oldFolder, newFolder, queue))
while True: #进度条
queue.get()
num += 1
culum = num / allNum
print("\r---复制的进度是:%d%%" % (culum * 100), end="")
time.sleep(1)
if num == allNum:
break
p.close()
p.join()
def main():
pool = Pool(10)
queue = Manager().Queue()
fr = open('url.txt', 'r')
lines = fr.readlines()
for i in lines:
url = remove_control_chars(i)
pool.apply_async(savePeopleInformation, args=(url, queue,))
allnum = len(lines)
num = 0
while True:
print queue.get()
num += 1
if num >= allnum:
fr.close()
break
def main():
time1 = time.time()
# 获取要复制的文件夹名称：
oldFolderName = input("请输入你要复制的文件夹名称：")
# 创建新的文件夹：
newFolderName = oldFolderName + '_复件'
# print(newFolderName)
os.mkdir(newFolderName)
# 获取old文件夹中的所有文件名字：
filesNames = os.listdir(oldFolderName)
# print(filesNames)
# 使用多线程的方式拷贝原文件夹中的文件到新的文件夹中。
pool = Pool(10)
queue = Manager().Queue()
for name in filesNames:
pool.apply_async(copyFileTask,
args=(name, oldFolderName, newFolderName, queue))
num = 0
allNum = len(filesNames)
while True:
queue.get()
num += 1
copyRate = num / allNum
print('\rcopy的进度是：%.2f%%' % (copyRate * 100), end='')
if num == allNum:
break
print('')
time2 = time.time()
print('运行时间为：%d s' % (time2 - time1))
pool.close()
def main():
#1.获取需要拷贝的文件夹
old_filename =input('请输入需要拷贝的文件名:')
#2.创建新的文件名
new_filename =old_filename+'[复件]'
os.mkdir(new_filename)
#3.获取文件夹中的文件列表
names =os.listdir(old_filename)
#4.创建进程池
pool =Pool(5)
#创建通讯
q =Manager().Queue()
#开始copy的时间
t1 = time.time()
#5.循环将进程放入进程池中
for name in names:
pool.apply_async(copyfile,(name,old_filename,new_filename,q))
#pool.close()
#pool.join()
num =0
allnum =len(names)
while num <allnum:
q.get()
num +=1
contern =num/allnum
print('\r\rcopy文件进度为:%.2f%%'%(contern*100),end='')
#结束copy的时间
t2 =time.time()
print('\n一共copy了%d个文件,总共耗时为:%.2f'%(allnum,(t2-t1)))
def main():
#0. 获取用于要copy的文件夹的名字
oldFileName = input('请输入文件夹的名字：')
# 创建一个文件夹
newFileName = oldFileName + '-复件'
os.mkdir(newFileName)
# 获取old文件夹中给所有的文件名字
fileNames = os.listdir(oldFileName)
# 使用多进程的方式copy 原文件夹中的文件到新的文件夹中
pool = Pool(5)
queue = Manager().Queue()
for fileName in fileNames:
pool.apply_async(copyFileTask, args=(fileName, oldFileName, newFileName, queue))
num = 0
allNum = len(fileNames)
while True:
queue.get()
num += 1
copyRate = num/allNum
print('\rcopy的进度是：%.2f %%'%(copyRate*100), end='')
if num == allNum:
break
pool.close()
pool.join()
def main():
#获取要拷贝的文件夹名字
oldFileName = input("请输入需要拷贝的文件夹名字：")
#创建一个文件夹
newFileName = oldFileName + "-复件"
#print(newFileName)
os.mkdir(newFileName)
#获取old文件夹中的名字
fileNames = os.listdir(oldFileName)
#print(fileNames)
#使用多进程的方式进行拷贝原文夹中的所有文件到新的文件夹中
pool = Pool(5)
queue = Manager().Queue()
for name in fileNames:
pool.apply_async(copyFileTask,
args=(name, oldFileName, newFileName, queue))
num = 0
allnum = len(fileNames)
while True:
queue.get()
num += 1
copyRate = num / allnum
print("拷贝的进度是%.2f%%" % (copyRate * 100))
if num == allnum:
break
print("已完成你需要拷贝的文件")
def main():
#0 获取要copy的文件夹的名字
oldFolderName = input("请输入文件夹的名字:")
#1 创建一个文件夹
newFolderName = oldFolderName + "-复件"
#print(newFolderName)
os.mkdir(newFolderName)
#2 获取old文件夹中的所有文件名字
filenames = os.listdir(oldFolderName)
#print(filenames)
#3 使用多进程的方式copy
pool = Pool(5)
queue = Manager().Queue()
for name in filenames:
pool.apply_async(copyFileTask,
args=(name, oldFolderName, newFolderName, queue))
num = 0
allNum = len(filenames)
while num < allNum:
queue.get()
num += 1
copyRate = num / allNum
print("\rcopy的进度是:%.2f%%" % (copyRate * 100), end="")
#pool.close()
#pool.join()
print("\nCopy Finished...Total num is %d" % num)
def main():
#0 get old folder's name and path
oldDirName = input("Please tell me the folder's name: ")
#print(oldDirName)
newDirName = oldDirName + "-copy"
#1 create new dir
os.mkdir(newDirName)
#2 get file list from old dir
fileNames = os.listdir(oldDirName)
# print(fileNames)
#3 copy files from old dir to new dir
# use multiprocessing to process large amounts of files
pool = Pool(5)
queue = Manager().Queue()
for name in fileNames:
pool.apply_async(copyFile, args=(name, oldDirName, newDirName, queue))
# pool.close()
# pool.join()
num = 0
numAll = len(fileNames)
while True:
queue.get()
num += 1
print("\r proceeing... %.2f%%" % (num / numAll * 100))
if num == numAll:
break
def main():
#获取需要备份的目录
oldFolderName = input("请输入需要备份的目录：")
#创建备份目录
newFolderName = oldFolderName + "-bak"
os.mkdir(newFolderName)
#获取源目录中的文件名列表
filenames = os.listdir(oldFolderName)
#通过多进程的方式拷贝文件
filenames_queue = Manager().Queue()
process_pool = Pool(5)
for name in filenames:
process_pool.apply_async(copy_file_content,
args=(name, oldFolderName, newFolderName,
filenames_queue))
num = 0
allnum = len(filenames)
while True:
filenames_queue.get()
num += 1
rate = num / allnum
print("\r当前备份进度为：%.2f%%" % (rate * 100), end="")
if num == allnum:
break
process_pool.close()
process_pool.join()
def __main__():
#1.获取要复制的文件夹名称
old_forder_name = input("请输入要复制的文件夹名字：")
#2.创建一个新的文件夹
new_forder_name = old_forder_name + "-复件"
os.mkdir(new_forder_name)
#print(new_forder_name)
#3.获取old forder里面所有文件的名
file_names = os.listdir(old_forder_name)
#print(file_names)
#4.使用多进程将源文件夹里面的文件 copy 到新文件夹中
pool = Pool(5)
queue = Manager().Queue()
for name in file_names:
pool.apply_async(copy_file,
args=(name, old_forder_name, new_forder_name, queue))
num = 0
all_num = len(file_names)
print("-" * 50)
while True:
queue.get()
num += 1
copy_rate = num / all_num
print("\r当前copy的进程是:%d %%" % (copy_rate * 100), end="")
if num == all_num:
print("")
break
def main():
# 取要copy的文件夹名字
old_folder_name = input("请输入要copy的文件夹名:")
# 创建一个文件夹
new_folder_name = old_folder_name + "-[复件]"
os.mkdir(new_folder_name)
# 获取老文件夹所有的文件名字
old_folder_list = os.listdir(old_folder_name)
# 使用多进程方式copy源文件所有文件到nw文件夹
pool = Pool(5)
queue = Manager().Queue()
for file_name in old_folder_list:
pool.apply_async(copy_file_task,
args=(file_name, old_folder_name, new_folder_name,
queue))
num = 0
all_num = len(old_folder_list)
while num < all_num:
queue.get()
num += 1
copy_rate = num / all_num
print("\rcopy的进度为:%.2f%%" % (copy_rate * 100), end="")
print("\n已完成copy.....")
pool.close()
pool.join()
def main():
#0.获取用户要copy的文件夹的名字
oldFolderName = input("请输入文件夹的名字")
#1,创建一个文件夹
newFoldername = oldFolderName + "-附件"
#print(newFoldername)
os.mkdir(newFoldername)
#2,获取old文件夹中的所有的文件名字
fileNames = os.listdir(oldFolderName)
#print(fileNames)
#3,使用多进程的方式copy 源文件夹中所有文件到新的文件夹中
pool = Pool(5)
queue = Manager().Queue()
for name in fileNames:
pool.apply_async(copyFileTask,
args=(name, oldFolderName, newFoldername, queue))
num = 0
allnum = len(fileNames)
while True:
queue.get()
num += 1
copyRate = num / allnum
print("\rcopy的进度是%.2f%%" % (copyRate * 100), end="")
if num == allnum:
print('已完成ＣＯＰＹ')
break
def main():
# 0.获取要copy的文件夹的名字
oldFolderName = input("请输入文件夹的名字：")
# 1. 创建一个文件夹
newFolderName = oldFolderName + "-复件"
# print(newFolderName)
os.mkdir(newFolderName)
# 2. 获取old文件夹所有文件名字
fileNames = os.listdir(oldFolderName)
print(fileNames)
# 3.使用多进程方式copy，原文件夹中的所有文件到新的文件夹中
pool = Pool(5)
queue = Manager().Queue()
for name in fileNames:
pool.apply_async(copyFileTask,
args=(name, oldFolderName, newFolderName, queue))
num = 0
allNum = len(fileNames)
while num <= allNum:
queue.get()
num += 1
copyRate = num / allNum
print("\rcopy的进度是：%。2f" % copyRate * 100, end="")
pool.close()
pool.join()
def main():
#0.获取用户要copy的文件夹的名字
oldFolderName = input('请输入文件夹的名字：')
#print(oldFolderName)
#1.创建一个文件夹
newFolderName = oldFolderName + '-复件'
#print(newFolderName)
os.mkdir(newFolderName)
#2.获取old文件夹中的所有的文件名字
fileNames = os.listdir(oldFolderName)
#print(fileNames)
#3.使用多进程的方式copy,原文件夹中的所有文件到新的文件夹中
pool = Pool(5)
queue = Manager().Queue()
for name in fileNames:
pool.apply_async(copyFileTask,
args=(name, oldFolderName, newFolderName, queue))
# pool.close()
# pool.join()
num = 0
allNum = len(fileNames)
while num < allNum:
queue.get()
num += 1
copyRate = num / allNum
print('\rcopy的进度是:%.2f%%' % (copyRate * 100), end='')
print('\n 已完成copy')
def main():
#获取用户copy文件夹的名字
oldfolername=input("文件夹：")
newfolername=oldfolername+"-复件"
#获取old文件夹中所有的文件
filenames=os.listdir(oldfolername)
#创建一个文件夹
os.mkdir(newfolername)
po=Pool(3)
q=Manager().Queue()
for name in filenames:
po.apply_async(copytask,(name,oldfolername,newfolername,q))
num=0
allnum=len(filenames)
while num<allnum:
q.get()
num=num+1
print("\r%.2f%%"%(num/allnum*100))
print("copy end")
def main():
try:
oldFolderName = input("请输入要复制的文件夹名:")
newFolderName = oldFolderName + "-复件"
#print("复制成功后的文件夹名为:%s"%newFolderName)
os.mkdir(newFolderName)
fileNames = os.listdir(oldFolderName)
except Exception as result:
if os.path.exists(newFolderName):
print("已捕获到异常信息如下:")
print(result)
pool = Pool(5)
queue = Manager().Queue()
for name in fileNames:
pool.apply_async(copyFileTask,
(name, oldFolderName, newFolderName, queue))
#pool.close()
#pool.join()
num = 0
allNum = len(fileNames)
while True:
queue.get()
num += 1
copyRate = num / allNum
print("\r复制的进度为:%.2f%%" % (copyRate * 100), end="")
time.sleep(0.1)
if num == allNum:
break
print("\n已完成复制...")
def main():
oldFolderName = "/home/lozzow/PycharmProjects/Python/Day_05"
newFolderName = oldFolderName + "-复件"
os.mkdir(newFolderName)
# 获取旧文件夹所有的文件名字
fileNames = os.listdir(oldFolderName) # 使用多进程的方式拷贝文件
pool = Pool(5)
queue = Manager().Queue()
for name in fileNames:
pool.apply_async(copyFileTask,
args=(
name,
oldFolderName,
newFolderName,
queue,
))
pool.close()
pool.join()
num = 0
allNum = len(fileNames)
while True:
queue.get()
num += 1
copyRate = num / allNum
print("\r copy的进度是%.2f%%" % (copyRate * 100), end="")
if num == allNum:
break
print("完成拷贝")
def main():
'''
多进程copy文件：
:return:
'''
#1.选择要拷贝的文件夹：
srcFolder = input("请输入要拷贝的文件夹：")
#2.创建一个同名文件夹+复件
newFolder = "复件"+srcFolder
os.mkdir(newFolder)
#3.获取文件夹中的文件
files = os.listdir(srcFolder)
#4.利用多进程进行拷贝
pool = Pool(6)
q = Manager().Queue()
#添加任务：
for file in files:
pool.apply_async(copy,args=(file,srcFolder,newFolder,q))
#pool.close()
#pool.join()
#主进程中计算copy进度：
num = 0
allNum = len(files)
while True:
q.get(True)
num+=1
copyRate = num / allNum
print("\nCopy的进度是：%.2f%%"%(copyRate*100),end="")
if num==allNum:
break
def multi_down(self):
q = Manager().Queue()
p = Pool(20) # 创建进程池
for num in range(self.max_pics):
p.apply_async(self.down, args=(
num,
q,
))
print("----start----")
print('下载...', end='')
p.close()
# p.join()
comle_count = 0
while True:
q.get() # 做阻塞用，保证每个线程完成后再做完成率计算
comle_count += 1
# print(comle_count)
# print(type(comle))
down_rate = comle_count / self.max_pics * 100
print('\r【%s】已经下载完成%0.2f%%(完成%d张，还剩%d张)...' %
(self.group_name, down_rate, comle_count,
self.max_pics - comle_count),
end=' ')
if down_rate >= 100:
print('\r【%s】已经下载完成(%d张).' % (self.group_name, self.max_pics))
break
print("----end----")
def main():
# o.获取文件夹的名字
oldFolderName = input("请输入您要拷贝的文件夹名：")
# 1.创建一个文件夹
newFolderName = oldFolderName + "-复件"
# print(newFolderName)
os.mkdir(newFolderName)
print("创建文件夹成功！")
# 2.获取old文件夹中所有的文件名
p = os.listdir(oldFolderName)
# print(p)
# 3.使用多进程拷贝
pool = Pool(5)
queue = Manager().Queue()
for name in p:
pool.apply_async(copyFileTask,
args=(name, oldFolderName, newFolderName, queue))
num = 0
allnum = len(p)
while True:
queue.get()
num += 1
copyRate = num / allnum
print("\rcopy的进度是：%.2f%%" % (copyRate * 100), end="")
if num == allnum:
break
# pool.close()
# pool.join()
print("\n已完成copy!")
'''
def main():
oldFileName = input("请输入旧的文件夹名字：")
newFileName = oldFileName + "复件"
#print(newFileName)
os.mkdir(newFileName)
filenames = os.listdir(oldFileName)
print(filenames)
pool = Pool(5)
queue = Manager().Queue()
for name in filenames:
print("文件名是%s" % name)
pool.apply_async(copyFile,
args=(name, oldFileName, newFileName, queue))
pool.close()
allNums = len(filenames)
num = 0
while True:
queue.get()
num += 1
copyrate = num / allNums * 100
print("\r%.2f%%" % copyrate)
time.sleep(0.01)
if num == allNums:
break
print("copy完成...")
def main():
#用户输入当前目录下需要拷贝的文件夹名
oldFolderName = input("请输入需要拷贝的文件夹的名字：")
#创建新文件夹
newFolderName = oldFolderName + "副本"
os.mkdir(newFolderName)
#获取old文件夹中所有的文件名字
fileNames = os.listdir(oldFolderName)
#多线程拷贝文件
pool = Pool(5)
queue = Manager().Queue()
for name in fileNames:
pool.apply_async(copyFile,args=(name,oldFolderName,newFolderName,queue))
#执行时判断进程，%在输出需要用两个，因为转义字符
num = 0
allFilenum = len(fileNames)
while num<allFilenum:
queue.get()
num += 1
copyRate = num/allFilenum
print("\r进度:%.2f%%"%(copyRate*100),end="")
# end="表示不换行"
print("已完成")
def main():
# 创建一个文件夹
new_folder_name = old_folder_name + "-复件"
# print(new_folder_name)
os.mkdir(new_folder_name)
# 获取old文件夹中的所有的文件的名字
file_names = os.listdir(old_folder_name)
# print(file_names)
# 使用多进程的方式copy原文件夹中的所有文件到新的文件夹中
pool = Pool(5)
queue = Manager().Queue()
for name in file_names:
pool.apply_async(copy_file_task,
args=(name, old_folder_name, new_folder_name,
queue)) # name后面要加逗号，此时才是元祖
num = 0
all_num = len(file_names)
while num <= all_num:
queue.get()
num += 1
copy_rate = num / all_num
print("\rcopy的进度是：%.2f" % (copy_rate * 100), end="")
print("\n已完成copy--------")
def main():
# 0. 输入要拷贝的文件夹名
oldFolderName = input("请输入文件夹名称 :")
# 1. 创建一个文件夹
newFolderName = oldFolderName + '-复件'
os.mkdir(newFolderName)
# 2. 获取 old 文件夹中的所有的文件名字
fileNames = os.listdir(oldFolderName)
print(fileNames)
# 3. 使用多进程的方式 copy 原文件夹中的所有文件
pool = Pool(5)
queue = Manager().Queue()
for name in fileNames:
pool.apply_async(copyFileTask,
args=(oldFolderName, newFolderName, name, queue))
num = 0
allNum = len(fileNames)
while num < allNum:
queue.get()
num += 1
copyRate = num / allNum
print("拷贝进度 %.2f %%" % (copyRate * 100), end='')
print("\n已经完成")
pool.close()
pool.join()
def main():
# 0. 获取要copy文件夹的名字
oldFolderName = input('请输入要拷贝的文件夹名字:')
newFolderName = oldFolderName + '-复件'
# 1. 创建一个文件夹
# print(newFolderName)
os.mkdir(newFolderName)
# 2. 获取old文件夹中的所有文件的名字
file_names = os.listdir(oldFolderName)
# print(file_names)
# 3. 使用多进程方式copy原文件夹中的文件到新文件夹中
po = Pool(5)
queue = Manager().Queue()
for name in file_names:
po.apply_async(copy_file_task,
args=(name, oldFolderName, newFolderName, queue))
num = 0
allNum = len(file_names)
while True:
queue.get()
num += 1
copyRate = num / allNum
print('\rcopy的进度是:%.2f%%' % (copyRate * 100), end='')
if num == allNum:
break
po.close()
po.join()
print('')
def main():
#0.获取用户要copy的文件夹的名字
oldFolderName = input("请输入文件夹的名字：")
#1.创建一个新的文件夹。
newFolderName = oldFolderName + "-复件"
#print(newFolderName)
os.mkdir(newFolderName)
#2.获取old文件夹中所有的文件的名字。
fileNames = os.listdir(oldFolderName)
#print(fileNames)
#3.使用多进程的方式copy原文件夹中的所有文件到新的文件夹中
pool = Pool(5)
queue = Manager().Queue()
for name in fileNames:
pool.apply_async(copyFileTask,
args=(name, oldFolderName, newFolderName, queue))
num = 0
allNum = len(fileNames)
while True:
queue.get()
num += 1
copyRate = num / allNum
print("\rcopy的进度是:%.2f%%" % (copyRate * 100), end="")
if num == allNum:
break
print("\n已完成c o p y!!!")
def main():
# 0. 获取用户要 copy 文件夹的名字
oldFolderName = input("请输入文件夹的名字:")
# 1. 创建一个文件夹
newFolderNmae = oldFolderName + "-复件"
#print(newFolderNmae)
os.mkdir(newFolderNmae)
# 2. 获取old文件夹中的所有文件的名字
fileNames = os.listdir(oldFolderName)
#print(fileNames)
# 3. 使用多进程的方式copy原文件夹中的所有文件到新的文件中
pool = Pool(5)
queue = Manager().Queue()
for name in fileNames:
pool.apply_async(copyFileTask,args=(name,oldFolderName,newFolderNmae,queue))
num = 0
allNum = len(fileNames)
copyRate = 0
while num<allNum:
queue.get()
num += 1
copyRate = num/allNum
print("\rcopy的进度是:%.2f%%"%(copyRate*100),end="")
print("\n已完成 copy....")
def run_finetuning(self, epochs):
"""
Run the train and test error evaluation and the backpropagation using conjugate gradient to optimize the
weights in order to make the DBN perform better.
@param epochs: The number of epochs to run the finetuning for.
"""
self.train_error = {}
self.test_error = {}
dbn.save_dbn(self.weight_matrices_added_biases, self.train_error, self.test_error, self.fout())
for epoch in range(epochs):
self.fout('Backprop: Epoch ' + str(epoch + 1))
result_queue = Manager().Queue()
w_queue = Manager().Queue()
# Start backprop process
proc = Process(target=self.backpropagation, args=(epoch, self.weight_matrices_added_biases, w_queue,))
proc.start()
# Start error eval processes
evaluations = []
evaluations.append((
self.weight_matrices_added_biases, epoch, True, data_processing.get_batch_list(training=True),
result_queue,
self.binary_output))
evaluations.append((
self.weight_matrices_added_biases, epoch, False, data_processing.get_batch_list(training=False),
result_queue, self.binary_output))
p = Pool(cpu_count())
p.map_async(error, evaluations)
p.close()
# Join multiple processes
p.join()
proc.join()
self.weight_matrices_added_biases = w_queue.get()
# Print and save error estimates
for e in range(2):
out = result_queue.get()
if out[0]:
self.train_error[epoch] = out[2]
self.fout(out[1])
else:
self.test_error[epoch] = out[2]
self.fout(out[1])
# Save DBN
dbn.save_dbn(self.weight_matrices_added_biases, self.train_error, self.test_error, self.fout())
def upload_test(self):
start_time = time.time()
q = Manager().Queue()
plist = []
for i in range(self.upload_user):
proc = Process(target=self.upload_one_user, args=(q,))
plist.append(proc)
for proc in plist:
proc.start()
for proc in plist:
proc.join()
while True:
if q.empty():
break
else:
if q.get() == 0:
self.upload_success += 1
else:
self.upload_fail += 1
use_time = time.time() - start_time
table = PrettyTable(["key", "value"])
table.add_row(["One File Size (M)", self.upload_file_size])
table.add_row(["All File Size (M)", self.upload_file_size * self.upload_number * self.upload_user])
table.add_row(["Process Count(user)", self.upload_user])
table.add_row(["Upload Count", self.upload_number * self.upload_user])
table.add_row(["Interval Time(s)", self.upload_time])
table.add_row(["Success count", self.upload_success])
table.add_row(["Fail count", self.upload_fail])
table.add_row(["Success ratio (%)",
(round(self.upload_success / float(self.upload_number * self.upload_user), 4) * 100)])
table.add_row(["Use time (s)", "%.2f" % use_time])
print table
class InMemoryStorage:
def __init__(self):
self.storage = Manager().dict()
def keys(self):
return self.storage.keys()
def set_val(self, key, val):
self.storage[key] = val
def get_val(self, key):
return self.storage[key]
def append_val(self, key, val):
# self.storage.setdefault(key, []).append(val) # 不适用于Manager()
# t=self.storage.setdefault(key, []) # !!!
# t.append(val)
# self.storage[key]=t
if key in self.storage:
self.storage[key]+=[val]
else:
self.storage[key]=[val]
def get_list(self, key):
return self.storage.get(key, [])
def _feed_helper(self, feed_name, feed_config):
feed_provider_name = feed_config['name']
feed_func = FEEDS_MAPPING.get(feed_provider_name)
LOG.debug("Geting feeds from '%s'", feed_name)
if not feed_func:
LOG.warning('Unknown feed provider %s', feed_provider_name)
return
feed_result = Manager().dict()
feed_func_timeout = feed_config.get('timeout') or feed_func.get('timeout')
p = Process(target=feed_func.get('func'), args=(feed_config, feed_result))
p.start()
p.join(feed_func_timeout)
if p.is_alive():
p.terminate()
status = 255
else:
status = p.exitcode
if status:
LOG.error("Fail to execute feed provider '%s' for '%s', exitcode '%d'", feed_provider_name, feed_name, status)
return None
else:
return feed_result.get('sensor_data', None)
def behavior_stat_to_db(param=Const.TRAIN, other=Const.OTHER, process_num=12):
global left_time
global X, user_item_pair
print 'Counting behavior......... '
x_data = Data(other.PROCESSED_DATA_PATH)
X = x_data.query(select_table_sql(param.X_TABLE), index=['user_id', 'item_id'])
X = X['behavior_type']
set_index = list(set(X.index))
user_item_pair = pd.DataFrame(set_index, columns=['user_id', 'item_id'])
print user_item_pair
user_item_len = len(user_item_pair)
test_num = user_item_len
core_num = process_num
# queue
queue = Manager().Queue()
q_result = Manager().Queue()
# split data to different processes
interval = test_num/core_num
left_time = interval
task_list = [i*interval for i in range(core_num)]
task_list.append(test_num)
####################################
start_CPU = time.clock()
start_time = time.time()
p = Pool(core_num+1)
for i in range(core_num):
p.apply_async(m_ui_pc, args=(task_list[i], task_list[i+1], queue))
p.apply_async(read_queue, args=(queue, q_result, core_num))
print('Waiting for all subprocesses done...')
p.close()
p.join()
print('All subprocesses done.')
value = q_result.get(True)
final = value.sort_index()
print final
end_CPU = time.clock()
end_time = time.time()
print '%f CPU second' % (end_CPU - start_CPU)
print '%f real second' % (end_time - start_time)
print
temp = Data(other.PROCESSED_DATA_PATH)
final.iloc[0:test_num].to_sql(param.X_STATI_BEHAVIOR_TABLE, temp.conn, if_exists='replace')
show = pd.read_sql_query(select_table_sql(param.X_STATI_BEHAVIOR_TABLE), temp.conn, index_col='index')
print show
x_data.close()
temp.close()
def do_git_srpm_import(opts, src_filepath, task, tmp_dir):
# should be run in the forked process, see:
# - https://bugzilla.redhat.com/show_bug.cgi?id=1253335
# - https://github.com/gitpython-developers/GitPython/issues/304
result_dict = Manager().dict()
proc = Process(target=actual_do_git_srpm_import, args=(opts, src_filepath, task, tmp_dir, result_dict))
proc.start()
proc.join()
if result_dict.get("hash") is None:
raise PackageImportException("Failed to import the source rpm: {}".format(src_filepath))
return str(result_dict["hash"])
def run(self, show_errors):
tests_path = "%s/%s" % (self.test_folder, self.test_subfolder)
test_total = 0
test_fails = 0
print("Running test suite: '%s'" % self._name)
print("==================================================")
q = Manager().Queue()
p_list = []
pool = Pool(processes=num_cores())
start_time = time.time()
# Loop through test cases (files)
for test_name in os.listdir(tests_path):
if not test_name.startswith("J"):
continue
test_path = os.path.join(tests_path, test_name)
test_total += 1
# Run joosc (i.e., run the test).
p = pool.apply_async(func=run_joosc, args=(self._joosc_options, test_path, q, ))
p_list.append(p)
for p in p_list:
ret = q.get(5)
if ret[0] < 0:
print("#\nUNEXPECTED ERROR: %s" % os.path.split(ret[1])[1])
elif self.is_correct_result(ret[0], os.path.split(ret[1])[1]) == False:
test_fails += 1
print("#\nTEST FAIL %d: %s" % (test_fails, os.path.split(ret[1])[1]))
if self.verbose:
print("OUTPUT:")
print("==================================================")
print(ret[2])
print("==================================================")
else:
sys.stdout.write('.')
sys.stdout.flush()
# Done tests
print("\n==================================================")
print("Test run successful. %s seconds" % (time.time() - start_time))
print("{} test(s) ran. {} test(s) failed.".format(test_total, test_fails))
def remote_call(self, rpc_code, *args, **kwargs):
"""
Make synchronous remote procedure calls on the Orchestrator.
:param rpc_code: RPC code.
:type rpc_code: int
:returns: Depends on the call.
:rtype: \\*
"""
# Create the response queue.
try:
response_queue = Manager().Queue()
# If the above fails we can assume the parent process is dead.
except:
exit(1)
# Send the RPC message.
self.send_msg(message_type = MessageType.MSG_TYPE_RPC,
message_code = rpc_code,
message_info = (response_queue, args, kwargs),
priority = MessagePriority.MSG_PRIORITY_HIGH)
# Get the response.
try:
raw_response = response_queue.get() # blocking call
# If the above fails we can assume the parent process is dead.
except:
exit(1)
# Return the response, or raise an exception on error.
success, response = raw_response
if not success:
exc_type, exc_value, tb_list = response
try:
sys.stderr.writelines(
format_exception_only(exc_type, exc_value) )
sys.stderr.writelines(
format_list(tb_list) )
except Exception:
pass
raise response[0], response[1]
return response
def upload_begin(self):
plist = []
q = Manager().Queue()
with open(self.list_path, 'r') as fp:
for i in fp:
if not i:
break
md5_crc32 = i.strip()[:41]
if md5_crc32 not in self.tmp_list and len(md5_crc32) == 41:
self.tmp_list.append(md5_crc32)
self.upload_num += 1
print self.upload_num
for md5_crc32_list in self.chunks(self.tmp_list, self.work_count):
proc = Process(target=self.upload_file, args=(q, md5_crc32_list,))
plist.append(proc)
for proc in plist:
proc.start()
for proc in plist:
proc.join()
while True:
if q.empty():
break
else:
r = q.get()
if r == 0:
self.success += 1
elif r == 1:
self.fail += 1
elif r == 2:
self.download_fail += 1
else:
pass
use_time = time.time() - self.start_time
table = PrettyTable(["key", "value"])
table.add_row(["Upload Count", len(set(self.tmp_list))])
table.add_row(["Success count", self.success])
table.add_row(["Fail count", self.fail])
table.add_row(["Download Fail", self.download_fail])
table.add_row(["Use time (s)", "%.2f" % use_time])
print table
def startServer(host, port, options):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
s.bind((host, port))
s.listen(0)
queue = Manager().Queue()
while True:
print "main: waiting for connection"
conn, addr = s.accept()
print 'main: Connected by', addr
data = conn.recv(1024)
print 'received port request'
p = Process(target = serverNewClient, args = (queue, options, ))
p.start()
while queue.empty():
time.sleep(0.05)
print "queue is still empty"
port = queue.get()
conn.sendall(str(port) + '\r\n')
print "assigned port %d to new client" % port
class Reader(object):
""" Play audio files.
"""
def __init__(self):
""" Player(text, **kwargs) -> Speak text.
"""
self._text = ''
# Setup the msg_dict for sending messages to the child process.
self._msg_dict = Manager().dict()
# Create a pipe for sending and receiving messages.
self._control_conn, self._player_conn = Pipe()
def __str__(self) -> str:
""" The information about the open file.
"""
return self._text
# Wait for the stream to open.
while 'info' not in self._msg_dict: pass
# Return the info string.
return self._msg_dict.get('info', '')
def __repr__(self) -> str:
""" __repr__ -> Returns a python expression to recreate this instance.
"""
repr_str = '' # "filename='%(_filename)s'" % self.__dict__
return '%s(%s)' % (self.__class__.__name__, repr_str)
def __enter__(self):
""" Provides the ability to use pythons with statement.
"""
try:
return self
except Exception as err:
print(err)
return None
def __exit__(self, exc_type, exc_value, traceback):
""" Stop playback when finished.
"""
try:
self.stop()
self._control_conn.close()
self._player_conn.close()
return not bool(exc_type)
except Exception as err:
print(err)
return False
def __del__(self):
""" Stop playback before deleting.
"""
pass
def __len__(self):
""" The length of the file if it has one.
"""
return self.length
def playing_wrapper(func):
""" Wrap methods and only call them if the stream is playing
"""
@functools_wraps(func)
def wrapper(self, *args, **kwargs):
""" Check if stream is playing and if it is then call func
otherwise print a message and exit.
"""
if not self.playing:
print("%(filename)s is not playing." % self._msg_dict)
return None
return func(self, *args, **kwargs)
return wrapper
def _play_proc(self, msg_dict: dict, pipe: Pipe):
""" Player process
"""
# Open the file to play.
with EspeakText(**msg_dict) as fileobj:
# Put the file info in msg_dict.
# msg_dict['info'] = str(fileobj)
msg_dict['length'] = fileobj.length
# Open an audio output device that can handle the data from
# fileobj.
# with AudioDevice(rate=22050, channels=1) as device:
device = AudioDevice(rate=22050, channels=1)
try:
# Set the default number of loops to infinite.
fileobj.loops = msg_dict.get('loops', -1)
# Initialize variable.
buf = b'\x00' * device.buffer_size
written = 0
# Loop until stopped or nothing read or written.
while msg_dict['playing'] and (buf or written):
# Keep playing if not paused.
if not msg_dict.get('paused', False):
# Re-open the device if it was closed.
if device.closed:
device = AudioDevice(rate=22050, channels=1)
# Read the next buffer full of data.
buf = fileobj.readline()
# Write buf.
written = device.write(buf)
else:
# Close the device when paused and sleep to
# open the audio for another process and
# save cpu cycles.
if not device.closed:
device.close()
time_sleep(0.05)
# Write a buffer of null bytes so the audio
# system can keep its buffer full.
# device.write(b'\x00' * device.buffer_size)
# Get and process any commands from the parent process.
if pipe.poll():
# Get the data into temp.
command = pipe.recv()
if 'getposition' in command:
pipe.send(fileobj.position)
elif 'setposition' in command:
fileobj.position = command['setposition']
except Exception as err:
print(err)
finally:
if not device.closed:
device.close()
# Set playing to False for the parent.
msg_dict['playing'] = False
def read(self, text: str, **kwargs):
""" Read the text.
"""
self._text = text
self._msg_dict['text'] = text
self._msg_dict.update(kwargs)
# After opening a new file stop the current one from playing.
self.stop()
# Pause it.
self.pause()
# Start it playing so seeking works.
self.play()
def play(self):
""" play() -> Start playback.
"""
if not self._msg_dict.get('playing', False):
# Set playing to True for the child process.
self._msg_dict['playing'] = True
# Open a new process to play a file in the background.
self._play_p = Process(target=self._play_proc,
args=(self._msg_dict, self._player_conn))
# Start the process.
self._play_p.start()
elif self._msg_dict.get('paused', True):
# Un-pause if paused.
self._msg_dict['paused'] = False
def stop(self):
""" stop() -> Stop playback.
"""
if self._msg_dict.get('playing', False):
# Stop playback.
self._msg_dict['playing'] = False
# Wait for the player process to stop.
self._play_p.join()
# Un-Pause.
self._msg_dict['paused'] = False
def pause(self):
""" pause() -> Pause playback.
"""
# Pause playback.
self._msg_dict['paused'] = True
@property
def paused(self) -> bool:
""" True if playback is paused.
"""
return self._msg_dict.get('paused', False)
@property
def playing(self) -> bool:
""" True if playing.
"""
return self._msg_dict.get('playing', False)
@property
def length(self) -> int:
""" Length of audio.
"""
return self._msg_dict.get('length', 0)
@property
@playing_wrapper
def position(self) -> int:
""" Current position.
"""
self._control_conn.send('getposition')
return self._control_conn.recv()
@position.setter
@playing_wrapper
def position(self, value: int):
""" Set the current position.
"""
self._control_conn.send({'setposition': int(value)})
@playing_wrapper
def tell(self) -> int:
""" tell -> Returns the current position.
"""
return self.position
def main():
import argparse
import logging
import os
import yaml
parser = argparse.ArgumentParser()
parser.add_argument('classifier')
parser.add_argument('--postprocess', action="store_true",
help='Run postprocessing, close blobs and remove noise')
parser.add_argument('videolist', help='A file listed all the videos to be indexed')
parser.add_argument('cores', type=int, help='Number of processes of paralellism')
args = parser.parse_args()
logging.basicConfig(level=logging.WARNING,
format="%(asctime)s - %(message)s")
classifier = zipfile.ZipFile(args.classifier)
global forest0, svmmodels, training_bosts, hist0
forest0, hist0, forest1, hist1, training_bosts, svmmodels, prior = \
load_from_classifier(classifier)
classifier.close()
KEY_FRAME_PERIOD = 2 # in seconds
#queue = Queue.Queue()
#data_queue = Queue.Queue()
queue = Manager().Queue()
data_queue = Manager().Queue()
for processes in [4]:
video_list = open(args.videolist, 'r')
log_file = open('statistics%d.txt' % processes, 'w')
fps = 0
fps_count = 0
for video_file in video_list:
video_file = video_file.strip()
name = os.path.splitext(video_file)[0]
file_path = os.path.join(VIDEO_RESOURCE, video_file)
log_file.write(file_path+"\n")
capture = cv.CaptureFromFile(file_path)
frame_rate = cv.GetCaptureProperty(capture, cv.CV_CAP_PROP_FPS)
total_frames = cv.GetCaptureProperty(capture, cv.CV_CAP_PROP_FRAME_COUNT)
log_file.write("frame rate: %.3f, total frames: %d\n" % (frame_rate, total_frames))
start_time0 = time.time()
key_frame_counter = 0
frame = cv.QueryFrame(capture)
os.makedirs("tmp")
while frame:
cv.SaveImage("tmp/" + name + "%d.png" % key_frame_counter, frame)
for i in xrange(int(KEY_FRAME_PERIOD * frame_rate)):
frame = cv.QueryFrame(capture)
key_frame_counter += 1
for i in xrange(key_frame_counter):
data_queue.put(i)
start_time = time.time()
ps = []
for group in xrange(processes):
p = Process(target = calculate_class, args=(name, queue, data_queue, ))
#p = threading.Thread(target = calculate_class, args=(name, queue, data_queue, ))
p.start()
ps.append(p)
for p in ps:
p.join()
elapse_time = time.time() - start_time
accuracy_file = open('360.txt', 'w')
while not queue.empty():
q_entry = queue.get()
frame_counter = q_entry[0]
ILP = q_entry[1]
accuracy_file.write('%d' % frame_counter)
for class_index, score in enumerate(ILP):
accuracy_file.write(',%.02f' % score)
accuracy_file.write('\n')
accuracy_file.close()
os.system("rm -rf tmp")
log_file.write("decoding time: %.2f, total time: %.2f, key frames: %d, frame per sec: %.3f\n" \
% (start_time - start_time0, elapse_time, key_frame_counter, key_frame_counter / elapse_time))
fps += key_frame_counter / elapse_time
fps_count += 1
#time.sleep(10)
video_list.close()
log_file.write("average fps: %.3f\n" % (fps/fps_count))
log_file.close()
def main():
import argparse
import logging
import os
import yaml
import cv
global processes
global forest0, svmmodels, training_bosts, hist0
parser = argparse.ArgumentParser()
parser.add_argument('classifier')
parser.add_argument('cores', type=int, help='Number of processes of paralellism')
parser.add_argument('--postprocess', action="store_true",
help='Run postprocessing, close blobs and remove noise')
args = parser.parse_args()
logging.basicConfig(level=logging.WARNING,
format="%(asctime)s - %(message)s")
classifier = zipfile.ZipFile(args.classifier)
forest0, hist0, forest1, hist1, training_bosts, svmmodels, prior = \
load_from_classifier(classifier)
classifier.close()
processes = args.cores
pool = Pool(processes = processes)
KEY_FRAME_PERIOD = 2 # in seconds
q = Manager().Queue()
total_frame = 0
new_flag = True
while True:
if not new_flag:
print "wait..."
time.sleep(1)
stream_list = get_list(CLOUDLET_RESOURCE, STREAM_RESOURCE)
new_flag = False
prev_stream = None
for stream in stream_list:
if stream.get("stream_description").find("denatured") == -1 or stream.get("stream_description").find("video") == -1 or stream.get("stream_description").find("pstf") != -1:
prev_stream = stream
continue
ILP_max = []
for i in xrange(len(CLASSES)):
ILP_max.append(0)
ILP_list = []
for i in xrange(len(CLASSES)):
ILP_list.append([])
path, name = stream.get("path").replace("mnt", "cloudletstore").rsplit('/', 1)
print os.path.join(path, name)
path_p, name_p = prev_stream.get("path").replace("mnt", "cloudletstore").rsplit('/', 1)
print os.path.join(path_p, name_p)
statinfo = os.stat(os.path.join(path_p, name_p))
prev_stream = stream
if statinfo.st_size == 0:
continue
new_flag = True
frame_rate = 30
capture = cv.CaptureFromFile(os.path.join(path, name))
frame_rate = cv.GetCaptureProperty(capture, cv.CV_CAP_PROP_FPS)
total_frames = cv.GetCaptureProperty(capture, cv.CV_CAP_PROP_FRAME_COUNT)
frame = cv.QueryFrame(capture)
print frame_rate, total_frames
print capture
start_time = time.time()
key_frame_counter_base = 0
while frame:
process_num = 0
while frame:
cv.SaveImage("indexing" + "%d.png" % process_num, frame)
for i in xrange(int(KEY_FRAME_PERIOD * frame_rate)):
frame = cv.QueryFrame(capture)
process_num += 1
if process_num == processes:
break
pool.map(calculate_class, [(q, x) for x in xrange(key_frame_counter_base, key_frame_counter_base + process_num)])
while not q.empty():
q_entry = q.get()
key_frame_counter = q_entry[0]
ILP = q_entry[1]
for class_index, score in enumerate(ILP):
if score > SCORE_THRESHOLD:
ILP_list[class_index].append((key_frame_counter * int(KEY_FRAME_PERIOD * frame_rate) + 1, score))
print (CLASSES[class_index], "%.02f" % score),
if score > ILP_max[class_index]:
ILP_max[class_index] = score
print
key_frame_counter_base += process_num
for class_index, frame_list in enumerate(ILP_list):
if not frame_list:
continue
frame_list_split = split_frame_list(frame_list, int(KEY_FRAME_PERIOD * frame_rate) * 2)
for frame_list, local_max_score in frame_list_split:
tag_entry = {}
tag_entry["tag"] = CLASSES[class_index] + ":%d" % (ILP_max[class_index] * 100)
tag_entry["tag_value"] = local_max_score
tag_entry["offset"] = frame_list[0] / frame_rate
tag_entry["duration"] = (frame_list[-1] - frame_list[0]) / frame_rate
tag_entry["segment"] = stream.get("segment")
print tag_entry
ret_dict = post(CLOUDLET_RESOURCE, TAG_RESOURCE, tag_entry)
if stream.get("stream_description").find("pstf") == -1:
stream_entry = {"stream_description": stream.get("stream_description") + "pstf;"}
ret_dict = put(CLOUDLET_RESOURCE, stream.get("resource_uri"), stream_entry)
elapse_time = time.time() - start_time
print "max score:"
print [(CLASSES[class_index], "%.02f" % score) for class_index, score in enumerate(ILP_max)]
print "total time: %.2f, key frames: %d, frame per sec: %.2f" \
% (elapse_time, key_frame_counter_base, key_frame_counter_base / elapse_time)
print
def simulate(self, cell):
run_param = self.run_param
amp = self.get_previous_amp()
self.run_param['init_amp'] = amp
# String to put before output to the terminal.
str_start = self.name
str_start += " "*(20 - len(self.name)) + ": "
if self.only_apply_electrode:
if self.verbose:
print str_start + "Only applying electrode."
soma_clamp_params = {
'idx': cell.somaidx,
'record_current': False,
'amp': amp, # [nA]
'dur': self.run_param['duration'], # [ms]
'delay': self.run_param['delay'], # [ms]
'pptype': self.run_param['pptype'],
}
stim = LFPy.StimIntElectrode(cell, **soma_clamp_params)
return
# Find a current that generates n spikes.
amp_low = 0
amp_high = 0
# Copy the run param so they can be given to the "sub" simulation.
sub_run_param = run_param.copy()
while True:
# Gather data from the sub process in a dictionary.
# manager = Manager()
sub_data = Manager().dict()
# Set the new amp.
sub_run_param['amp'] = amp
# Run the "sub" simulation.
target = self.simulate_sub
args = (cell, sub_data, sub_run_param)
process = Process(target=target, args=args)
process.start()
process.join()
spike_cnt = sub_data.get('spike_cnt', 0)
if self.verbose:
print str_start + 'Found {} spikes at current {} nA.'.format(spike_cnt,
amp)
# Change the amplitude according to the spike cnt.
if spike_cnt == run_param['spikes']:
break
elif spike_cnt < run_param['spikes']:
amp_low = amp
elif spike_cnt > run_param['spikes']:
amp_high = amp
# Increase the amp until we have more than the desired number of spikes.
if amp_high == 0:
amp = 1.25 * amp
continue
amp = 0.5 * (amp_high + amp_low)
if amp < 1e-4 or amp > 1e4:
print str_start + 'Curent amplitude is above or under threshold, finishing.'
return
# Give the data back.
self.data['amp'] = amp
self.data['spike_cnt'] = spike_cnt
self.data['dt'] = cell.timeres_NEURON
self.data['stimulus_i'] = sub_data['stimulus_i']
self.data['soma_v'] = sub_data['soma_v']
self.data['soma_t'] = sub_data['soma_t']
if self.apply_electrode_at_finish:
soma_clamp_params = {
'idx': cell.somaidx,
'record_current': True,
'amp': self.data['amp'], # [nA]
'dur': self.run_param['duration'], # [ms]
'delay': self.run_param['delay'], # [ms]
'pptype': self.run_param['pptype'],
}
stim = LFPy.StimIntElectrode(cell, **soma_clamp_params)
class TestVariant(object):
def __init__(self, name, compile_flags=[]):
self.name = name
self.compile_flags = \
['-WERExceptionSupport', '-ExtendedErrorStackForTestHost',
'-BaselineMode'] + compile_flags
self.tags = tags.copy()
self.not_tags = not_tags.union(
['{}_{}'.format(x, name) for x in ('fails','exclude')])
self.msg_queue = Manager().Queue() # messages from multi processes
self.test_result = TestResult()
self._print_lines = [] # _print lines buffer
self._last_len = 0
# check if this test variant should run a given test
def _should_test(self, test):
tags = split_tags(test.get('tags'))
if not tags.isdisjoint(self.not_tags):
return False
if self.tags and not self.tags.issubset(tags):
return False
if not_compile_flags: # exclude unsupported compile-flags if any
flags = test.get('compile-flags')
if flags and \
not not_compile_flags.isdisjoint(flags.lower().split()):
return False
return True
# print output from multi-process run, to be sent with result message
def _print(self, line):
self._print_lines.append(str(line))
# queue a test result from multi-process runs
def _log_result(self, test, fail):
output = '\n'.join(self._print_lines) # collect buffered _print output
self._print_lines = []
self.msg_queue.put((test.filename, fail, test.elapsed_time, output))
# (on main process) process one queued message
def _process_msg(self, msg):
filename, fail, elapsed_time, output = msg
self.test_result.log(filename, fail=fail)
line = '[{}/{} {:4.2f}] {} -> {}'.format(
self.test_result.total_count(),
self.test_count,
elapsed_time,
'Failed' if fail else 'Passed',
self._short_name(filename))
padding = self._last_len - len(line)
print(line + ' ' * padding, end='\n' if fail else '\r')
log_message(line)
self._last_len = len(line) if not fail else 0
if len(output) > 0:
print_and_log(output)
# get a shorter test file path for display only
def _short_name(self, filename):
folder = os.path.basename(os.path.dirname(filename))
return os.path.join(folder, os.path.basename(filename))
# (on main process) wait and process one queued message
def _process_one_msg(self):
self._process_msg(self.msg_queue.get())
# log a failed test with details
def _show_failed(self, test, flags, exit_code, output,
expected_output=None, timedout=False):
if timedout:
self._print('ERROR: Test timed out!')
self._print('{} {} {}'.format(binary, ' '.join(flags), test.filename))
if expected_output == None or timedout:
self._print("\nOutput:")
self._print("----------------------------")
self._print(output)
self._print("----------------------------")
else:
lst_output = output.split(b'\n')
lst_expected = expected_output.split(b'\n')
ln = min(len(lst_output), len(lst_expected))
for i in range(0, ln):
if lst_output[i] != lst_expected[i]:
self._print("Output: (at line " + str(i) + ")")
self._print("----------------------------")
self._print(lst_output[i])
self._print("----------------------------")
self._print("Expected Output:")
self._print("----------------------------")
self._print(lst_expected[i])
self._print("----------------------------")
break
self._print("exit code: {}".format(exit_code))
self._log_result(test, fail=True)
# temp: try find real file name on hard drive if case mismatch
def _check_file(self, folder, filename):
path = os.path.join(folder, filename)
if os.path.isfile(path):
return path # file exists on disk
filename_lower = filename.lower()
files = os.listdir(folder)
for i in range(len(files)):
if files[i].lower() == filename_lower:
self._print('\nWARNING: {} should be {}\n'.format(
path, files[i]))
return os.path.join(folder, files[i])
# cann't find the file, just return the path and let it error out
return path
# run one test under this variant
def test_one(self, test):
try:
test.start()
self._run_one_test(test)
except Exception:
test.done()
self._print(traceback.format_exc())
self._log_result(test, fail=True)
# internally perform one test run
def _run_one_test(self, test):
folder = test.folder
js_file = test.filename = self._check_file(folder, test.files)
js_output = b''
working_path = os.path.dirname(js_file)
flags = test.get('compile-flags')
flags = self.compile_flags + (flags.split() if flags else [])
cmd = [binary] + flags + [os.path.basename(js_file)]
test.start()
proc = SP.Popen(cmd, stdout=SP.PIPE, stderr=SP.STDOUT, cwd=working_path)
timeout_data = [proc, False]
def timeout_func(timeout_data):
timeout_data[0].kill()
timeout_data[1] = True
timeout = test.get('timeout', args.timeout) # test override or default
timer = Timer(timeout, timeout_func, [timeout_data])
try:
timer.start()
js_output = normalize_new_line(proc.communicate()[0])
exit_code = proc.wait()
finally:
timer.cancel()
test.done()
# shared _show_failed args
fail_args = { 'test': test, 'flags': flags,
'exit_code': exit_code, 'output': js_output };
# check timed out
if (timeout_data[1]):
return self._show_failed(timedout=True, **fail_args)
# check ch failed
if exit_code != 0:
return self._show_failed(**fail_args)
# check output
if 'baseline' not in test:
# output lines must be 'pass' or 'passed' or empty
lines = (line.lower() for line in js_output.split(b'\n'))
if any(line != b'' and line != b'pass' and line != b'passed'
for line in lines):
return self._show_failed(**fail_args)
else:
baseline = test.get('baseline')
if baseline:
# perform baseline comparison
baseline = self._check_file(working_path, baseline)
with open(baseline, 'rb') as bs_file:
baseline_output = bs_file.read()
# Cleanup carriage return
# todo: remove carriage return at the end of the line
# or better fix ch to output same on all platforms
expected_output = normalize_new_line(baseline_output)
if expected_output != js_output:
return self._show_failed(
expected_output=expected_output, **fail_args)
# passed
self._log_result(test, fail=False)
# run tests under this variant, using given multiprocessing Pool
def run(self, tests, pool):
print_and_log('\n############# Starting {} variant #############'\
.format(self.name))
if self.tags:
print_and_log(' tags: {}'.format(self.tags))
for x in self.not_tags:
print_and_log(' exclude: {}'.format(x))
print_and_log()
# filter tests to run
tests = [x for x in tests if self._should_test(x)]
self.test_count = len(tests)
# run tests in parallel
result = pool.map_async(run_one, [(self,test) for test in tests])
while self.test_result.total_count() != self.test_count:
self._process_one_msg()
# print test result summary
def print_summary(self):
print_and_log('\n######## Logs for {} variant ########'\
.format(self.name))
for folder, result in sorted(self.test_result.folders.items()):
print_and_log('{}: {}'.format(folder, result))
print_and_log("----------------------------")
print_and_log('Total: {}'.format(self.test_result))
FPS_clock = pygame.time.Clock()
game_state = state.GameState()
game_gui = gui.GUI(game_state)
game_event_handler = event_handler.EventLogic(game_state, game_gui)
game_gui.add_handler(game_event_handler)
game_gui.draw(game_state.get_state())
pygame.display.update()
commandQueue = Manager().Queue()
listeningProcess = Process(target=voice_listener, args=(game_event_handler, commandQueue,))
while True:
game_gui.draw(game_state.get_state())
game_event_handler.event_handler()
if game_state.get_state() == "SSH season voice mode" or game_state.get_state() == "Web season voice mode":
if not game_event_handler.queue.empty():
val = game_event_handler.queue.get()
if val:
listeningProcess.start()
else:
listeningProcess.terminate()
listeningProcess.join()
listeningProcess = Process(target=voice_listener, args=(game_event_handler, commandQueue,))
if not commandQueue.empty():
voice_command = commandQueue.get()
try:
game_event_handler.pipi.say(voice_command %
game_gui.bool_to_text[str(game_gui.light_to_string[voice_command])])
except KeyError:
pass
pygame.display.update()
FPS_clock.tick(30)
####################################
start_CPU = time.clock()
start_time = time.time()
p = Pool(core_num+1)
result = []
for i in range(core_num):
p.apply_async(m_ui_pc, args=(task_list[i], task_list[i+1], queue))
p.apply_async(read_queue, args=(queue, q_result, core_num))
print('Waiting for all subprocesses done...')
p.close()
p.join()
print('All subprocesses done.')
value = q_result.get(True)
final = value.sort_index()
print final
end_CPU = time.clock()
end_time = time.time()
print '%f CPU second' % (end_CPU - start_CPU)
print '%f real second' % (end_time - start_time)
print
temp = Data('./data/train_behavior.db')
final.iloc[0:test_num].to_sql('user_item_behavior', temp.conn, if_exists='replace')
show = pd.read_sql_query('select* from user_item_behavior', temp.conn, index_col='index')
print show
all_data.close()
class WorkerPool(object):
"""
Manages a bunch of worker processes.
"""
def __init__(self, workers=4):
if workers < 1:
raise ValueError(_("at least one worker is required"))
# A "worker" is simply a tuple consisting of a Process object
# and our end of a pipe. Each worker is always adressed with
# it's "worker id" (wid): That's the index of the tuple in
# self.workers.
self.workers = []
# Lists of wids. idle_workers are those that are marked
# explicitly as idle (don't confuse this with workers that
# aren't processing a job right now).
self.idle_workers = []
self.workers_alive = list(range(workers))
# We don't need to know *which* worker is currently processing a
# job. We only need to know how many there are.
self.jobs_open = 0
# The public message queue. Workers ask for jobs here, report
# finished work and log items.
# Note: There's (at least) two ways to organize a pool like
# this. One is to only open a pipe to each worker. Then, you can
# use select() to see which pipe can be read from. Problem is,
# this is not really supported by the multiprocessing module;
# you have to dig into the internals and that's ugly. So, we go
# for another option: A managed queue. Multiple processes can
# write to it (all the workers do) and the parent can read from
# it. However, this is only feasible when workers must talk to
# the parent. The parent can't talk to the workers using this
# queue. Thus, we still need a dedicated pipe to each worker
# (see below).
self.messages = Manager().Queue()
for i in range(workers):
(parent_conn, child_conn) = Pipe()
p = Process(target=_worker_process,
args=(i, self.messages, child_conn))
p.start()
self.workers.append((p, parent_conn))
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.shutdown()
def get_event(self):
"""
Blocks until a message from a worker is received.
"""
msg = self.messages.get()
if msg['msg'] == 'FINISHED_WORK':
self.jobs_open -= 1
# check for exception in child process and raise it
# here in the parent
if not msg['traceback'] is None:
raise WorkerException(
msg['exception_task_id'],
msg['exception'],
msg['traceback'],
)
elif msg['msg'] == 'LOG_ENTRY':
LOG.handle(msg['log_entry'])
return msg
def start_task(self, wid, target, task_id=None, args=None, kwargs=None):
"""
wid id of the worker to use
target any callable (includes bound methods)
task_id something to remember this worker by
args list of positional arguments passed to target
kwargs dictionary of keyword arguments passed to target
"""
if args is None:
args = []
else:
args = list(args)
if kwargs is None:
kwargs = {}
if ismethod(target):
target_obj = target.__self__
target = target.__name__
else:
target_obj = None
(process, pipe) = self.workers[wid]
pipe.send({
'msg': 'RUN',
'task_id': task_id,
'target': target,
'target_obj': target_obj,
'args': args,
'kwargs': kwargs,
})
self.jobs_open += 1
def mark_idle(self, wid):
"""
Mark a worker as "idle".
"""
# We don't really need to do something here. The worker will
# simply keep blocking at his "pipe.read()". Just store his id
# so we can answer him later.
self.idle_workers.append(wid)
def quit(self, wid):
"""
Shutdown a worker.
"""
(process, pipe) = self.workers[wid]
try:
pipe.send({'msg': 'DIE'})
except IOError:
pass
pipe.close()
process.join(JOIN_TIMEOUT)
if process.is_alive():
LOG.warn(_(
"worker process with PID {pid} didn't join "
"within {time} seconds, terminating...").format(
pid=process.pid,
time=JOIN_TIMEOUT,
)
)
process.terminate()
self.workers_alive.remove(wid)
def shutdown(self):
"""
Shutdown all workers.
"""
while self.workers_alive:
self.quit(self.workers_alive[0])
def activate_idle_workers(self):
"""
Tell all idle workers to ask for work again.
"""
for wid in self.idle_workers:
# Send a noop to this worker. He will simply ask for new
# work again.
(process, pipe) = self.workers[wid]
pipe.send({'msg': 'NOOP'})
self.idle_workers = []
def keep_running(self):
"""
Returns True if this pool is not ready to die.
"""
return self.jobs_open > 0 or self.workers_alive
class TestVariant(object):
def __init__(self, name, compile_flags=[], variant_not_tags=[]):
self.name = name
self.compile_flags = \
['-WERExceptionSupport', '-ExtendedErrorStackForTestHost',
'-BaselineMode'] + compile_flags
self._compile_flags_has_expansion = self._has_expansion(compile_flags)
self.tags = tags.copy()
self.not_tags = not_tags.union(variant_not_tags).union(
['{}_{}'.format(x, name) for x in ('fails','exclude')])
self.msg_queue = Manager().Queue() # messages from multi processes
self.test_result = TestResult()
self.test_count = 0
self._print_lines = [] # _print lines buffer
self._last_len = 0
@staticmethod
def _has_expansion(flags):
return any(re.match('.*\${.*}', f) for f in flags)
@staticmethod
def _expand(flag, test):
return re.sub('\${id}', str(test.id), flag)
def _expand_compile_flags(self, test):
if self._compile_flags_has_expansion:
return [self._expand(flag, test) for flag in self.compile_flags]
return self.compile_flags
# check if this test variant should run a given test
def _should_test(self, test):
tags = split_tags(test.get('tags'))
if not tags.isdisjoint(self.not_tags):
return False
if self.tags and not self.tags.issubset(tags):
return False
if not_compile_flags: # exclude unsupported compile-flags if any
flags = test.get('compile-flags')
if flags and \
not not_compile_flags.isdisjoint(flags.lower().split()):
return False
return True
# print output from multi-process run, to be sent with result message
def _print(self, line):
self._print_lines.append(str(line))
# queue a test result from multi-process runs
def _log_result(self, test, fail):
output = '\n'.join(self._print_lines) # collect buffered _print output
self._print_lines = []
self.msg_queue.put((test.filename, fail, test.elapsed_time, output))
# (on main process) process one queued message
def _process_msg(self, msg):
filename, fail, elapsed_time, output = msg
self.test_result.log(filename, fail=fail)
line = '[{}/{} {:4.2f}] {} -> {}'.format(
self.test_result.total_count(),
self.test_count,
elapsed_time,
'Failed' if fail else 'Passed',
self._short_name(filename))
padding = self._last_len - len(line)
print(line + ' ' * padding, end='\n' if fail else '\r')
log_message(line)
self._last_len = len(line) if not fail else 0
if len(output) > 0:
print_and_log(output)
# get a shorter test file path for display only
def _short_name(self, filename):
folder = os.path.basename(os.path.dirname(filename))
return os.path.join(folder, os.path.basename(filename))
# (on main process) wait and process one queued message
def _process_one_msg(self):
self._process_msg(self.msg_queue.get())
# log a failed test with details
def _show_failed(self, test, flags, exit_code, output,
expected_output=None, timedout=False):
if timedout:
if warn_on_timeout:
self._print('WARNING: Test timed out!')
else:
self._print('ERROR: Test timed out!')
self._print('{} {} {}'.format(binary, ' '.join(flags), test.filename))
if expected_output == None or timedout:
self._print("\nOutput:")
self._print("----------------------------")
self._print(output.decode('utf-8'))
self._print("----------------------------")
else:
lst_output = output.split(b'\n')
lst_expected = expected_output.split(b'\n')
ln = min(len(lst_output), len(lst_expected))
for i in range(0, ln):
if lst_output[i] != lst_expected[i]:
self._print("Output: (at line " + str(i+1) + ")")
self._print("----------------------------")
self._print(lst_output[i])
self._print("----------------------------")
self._print("Expected Output:")
self._print("----------------------------")
self._print(lst_expected[i])
self._print("----------------------------")
break
self._print("exit code: {}".format(exit_code))
if warn_on_timeout and timedout:
self._log_result(test, fail=False)
else:
self._log_result(test, fail=True)
# temp: try find real file name on hard drive if case mismatch
def _check_file(self, folder, filename):
path = os.path.join(folder, filename)
if os.path.isfile(path):
return path # file exists on disk
filename_lower = filename.lower()
files = os.listdir(folder)
for i in range(len(files)):
if files[i].lower() == filename_lower:
self._print('\nWARNING: {} should be {}\n'.format(
path, files[i]))
return os.path.join(folder, files[i])
# cann't find the file, just return the path and let it error out
return path
# run one test under this variant
def test_one(self, test):
try:
test.start()
self._run_one_test(test)
except Exception:
test.done()
self._print(traceback.format_exc())
self._log_result(test, fail=True)
# internally perform one test run
def _run_one_test(self, test):
folder = test.folder
js_file = test.filename = self._check_file(folder, test.files)
js_output = b''
working_path = os.path.dirname(js_file)
flags = test.get('compile-flags') or ''
flags = self._expand_compile_flags(test) + \
args.flags.split() + \
flags.split()
if test.get('custom-config-file') != None:
flags = ['-CustomConfigFile:' + test.get('custom-config-file')]
if args.lldb == None:
cmd = [binary] + flags + [os.path.basename(js_file)]
else:
lldb_file = open(working_path + '/' + os.path.basename(js_file) + '.lldb.cmd', 'w')
lldb_command = ['run'] + flags + [os.path.basename(js_file)]
lldb_file.write(' '.join([str(s) for s in lldb_command]));
lldb_file.close()
cmd = ['lldb'] + [binary] + ['-s'] + [os.path.basename(js_file) + '.lldb.cmd'] + ['-o bt'] + ['-b']
test.start()
proc = SP.Popen(cmd, stdout=SP.PIPE, stderr=SP.STDOUT, cwd=working_path)
timeout_data = [proc, False]
def timeout_func(timeout_data):
timeout_data[0].kill()
timeout_data[1] = True
timeout = test.get('timeout', args.timeout) # test override or default
timer = Timer(timeout, timeout_func, [timeout_data])
skip_baseline_match = False
try:
timer.start()
js_output = normalize_new_line(proc.communicate()[0])
exit_code = proc.wait()
# if -lldb was set; check if test was crashed before corrupting the output
search_for = " exited with status = 0 (0x00000000)"
if args.lldb != None and exit_code == 0 and js_output.index(search_for) > 0:
js_output = js_output[0:js_output.index(search_for)]
exit_pos = js_output.rfind('\nProcess ')
if exit_pos > len(js_output) - 20: # if [Process ????? <seach for>]
if 'baseline' not in test:
js_output = "pass"
else:
skip_baseline_match = True
finally:
timer.cancel()
test.done()
# shared _show_failed args
fail_args = { 'test': test, 'flags': flags,
'exit_code': exit_code, 'output': js_output }
# check timed out
if (timeout_data[1]):
return self._show_failed(timedout=True, **fail_args)
# check ch failed
if exit_code != 0:
return self._show_failed(**fail_args)
# check output
if 'baseline' not in test:
# output lines must be 'pass' or 'passed' or empty
lines = (line.lower() for line in js_output.split(b'\n'))
if any(line != b'' and line != b'pass' and line != b'passed'
for line in lines):
return self._show_failed(**fail_args)
else:
baseline = test.get('baseline')
if not skip_baseline_match and baseline:
# perform baseline comparison
baseline = self._check_file(folder, baseline)
with open(baseline, 'rb') as bs_file:
baseline_output = bs_file.read()
# Cleanup carriage return
# todo: remove carriage return at the end of the line
# or better fix ch to output same on all platforms
expected_output = normalize_new_line(baseline_output)
if expected_output != js_output:
return self._show_failed(
expected_output=expected_output, **fail_args)
# passed
if verbose:
self._print('{} {} {}'.format(binary, ' '.join(flags), test.filename))
self._log_result(test, fail=False)
# run tests under this variant, using given multiprocessing Pool
def _run(self, tests, pool):
print_and_log('\n############# Starting {} variant #############'\
.format(self.name))
if self.tags:
print_and_log(' tags: {}'.format(self.tags))
for x in self.not_tags:
print_and_log(' exclude: {}'.format(x))
print_and_log()
# filter tests to run
tests = [x for x in tests if self._should_test(x)]
self.test_count += len(tests)
# run tests in parallel
result = pool.map_async(run_one, [(self,test) for test in tests])
while self.test_result.total_count() != self.test_count:
self._process_one_msg()
# print test result summary
def print_summary(self):
print_and_log('\n######## Logs for {} variant ########'\
.format(self.name))
for folder, result in sorted(self.test_result.folders.items()):
print_and_log('{}: {}'.format(folder, result))
print_and_log("----------------------------")
print_and_log('Total: {}'.format(self.test_result))
# run all tests from testLoader
def run(self, testLoader, pool, sequential_pool):
tests, sequential_tests = [], []
for folder in testLoader.folders():
if folder.tags.isdisjoint(self.not_tags):
dest = tests if not folder.is_sequential else sequential_tests
dest += folder.tests
if tests:
self._run(tests, pool)
if sequential_tests:
self._run(sequential_tests, sequential_pool)
calculation_params = []
for _ in range(0, tasks):
calculation_params.append([])
for index in range(0, precision):
calculation_params[index % tasks].append(index)
queue = Manager().Queue(maxsize=tasks)
start_time = timer()
processes = []
for thread_index in range(0, len(calculation_params)):
p = Process(target=calculate, args=(calculation_params[thread_index],
thread_index, queue, quiet))
processes.append(p)
p.start()
result = 0
for _ in range(0, tasks):
result += queue.get()
for p in processes:
p.join()
time = timer() - start_time
message = 'Using %s tasks\nElapsed time: %s seconds' % (tasks,
time)
if not quiet:
print(message)
if output_file_name:
with open(output_file_name, 'a') as file:
file.write('%s\n' % unicode(str(time), "utf-8"))
####################################### # 再生 ####################################### # 現在再生中の音声のインデックス nowplaying = -1 # 現在再生中のaplayコマンドのProcess playing = None # 進捗を確認しつつ音声を読み上げる while ( not r.ready() ) or ( nowplaying != len(arg) ) or ( playing is not None ): time.sleep(0.5) # 音声合成の終了報告があるかキューを確認する。 for _ in range(queue.qsize()): compiled_index = queue.get() l[compiled_index] = 1 # 再生できるならしてみる？ if nowplaying < len(arg): if playing is None: if l[nowplaying + 1] == 1: # まとめてWAVファイルを指定できるときはする listindex = list() while l[nowplaying + 1] == 1: nowplaying += 1 listindex.append(nowplaying) _print( "DEBUG: しゃべるよ！[%s]" % str(listindex) ) playing = play_wav(listindex) elif l[nowplaying + 1] == 0: _print( "DEBUG: 音声合成の完了待ちです！" ) else:
class MPResult(object):
"""
Sync result between processes
"""
MATCH = {} # id -> instance
def __init__(self, result):
from multiprocessing import Manager
# Test result instance
self.result = result
# Result queue
self.queue = Manager().Queue()
def __getattr__(self, item):
return getattr(self.result, item)
@staticmethod
def pack_result_storage(storage):
"""
Pack result from storage
"""
return [(get_master_id(s[0]), s[1]) for s in storage]
def unpack_result_storage(self, storage):
"""
Unpack result from storage
"""
unpack_storage = []
for master_id, message in storage:
unpack_storage.append(
(self.MATCH[master_id], message),
)
return unpack_storage
def match(self, suite):
"""
Match id of master process to instance
"""
self.MATCH[get_suite_master_id(suite)] = suite
def match(s):
for o in s:
if isinstance(o, BaseSuite):
self.MATCH[get_suite_master_id(o)] = o
match(o)
else:
self.MATCH[get_case_master_id(o)] = o
match(suite)
def save_result(self):
"""
Save result in queue
"""
self.queue.put(
(
(
self.pack_result_storage(self.result.errors),
self.pack_result_storage(self.result.skipped),
self.pack_result_storage(self.result.failures),
),
self.result.testsRun,
),
)
def make_result(self):
"""
Merge result from queue to result instance
"""
while not self.queue.empty():
(errors, skipped, failures), run_tests = self.queue.get()
self.result.errors.extend(self.unpack_result_storage(errors))
self.result.skipped.extend(self.unpack_result_storage(skipped))
self.result.failures.extend(self.unpack_result_storage(failures))
self.result.testsRun += run_tests
'''
main ,start here
'''
now = time.time()
amazon = AmazonSpider()
html = amazon.getHTML(amazon.url)
time.sleep(5)
amazon.parseBase(html)
pool = Pool(5)
q = Manager().Queue()
for color_id in amazon.color_list:
pool.apply_async(start, args = (color_id, amazon.size_list, amazon.url, q, ))
pool.close()
pool.join()
while q.qsize() != 0:
print q.get(False)
end = time.time()
print (end-now)
amazon.driver.quit()
# for color_id in amazon.color_list:
# xpath = "//li[@id='"+color_id+"']"
# amazon.driver.find_element_by_xpath(xpath).click()
# time.sleep(2)
# for size in amazon.size_list:
# if size == 'native_size_name_-1':
# continue
# if 'U' in amazon.driver.find_element_by_xpath("//option[@id='"+size+"']").get_attribute('class'):
# continue
# else:
# amazon.driver.find_element_by_xpath("//option[@id='"+size+"']").click()
# time.sleep(2)
def launch_expeditions( self , task_request_list , moon_name_list=None ):
global expedition
# ---[ 1 ]------------------------------------------------------
self.log.show( 'Checking Moon list sent by user' )
working_moons = []
if not moon_name_list :
self.log.show( 'Traveling to available Moons on Orbit' )
working_moons = self.orbit.values()
else :
self.log.show( 'Traveling to ' + str( moon_name_list ) )
working_moons = [ self.orbit.get_moon( moon_name ) for moon_name in moon_name_list ]
# ---[ 2 ]------------------------------------------------------
self.log.show( 'Build Thread-safe Queues with no maximum size' )
recv_queue = Manager().Queue( )#len(task_request_list) )
send_queue = Manager().Queue( )#len(task_request_list) )
# ---[ 3 ]------------------------------------------------------
self.log.show( 'Enqueue tasks on "send_queue" object' )
for task_obj in task_request_list :
send_queue.put_nowait( str(task_obj) ) # "Normal" Objects are note thread safe!
self.log.show( 'send_queue = ' + str(send_queue.qsize())+'/'+str(len(task_request_list)) + 'tasks')
# ---[ 4 ]------------------------------------------------------
self.log.show( 'Starting up Process Pool' )
pool = Pool(processes=len(working_moons))
for moon in working_moons :
#running_expeditions.append( Process( target=expedition , args=(self.name , moon.name , moon.ip , moon.port , taskrequest_queue , taskresponse_queue, ) ) ) # Process Object
pool.apply_async( func=expedition , args=(self.name , moon.name , moon.ip , moon.port , send_queue , recv_queue , ) )
# ---[ 5 ]------------------------------------------------------
pool.close()
pool.join()
self.log.show( 'recv_queue = '+ str(recv_queue.qsize())+'/'+str(len(task_request_list)) + 'tasks' )
tmp = []
while not recv_queue.empty() :
tmp.append( recv_queue.get() )
self.log.show( 'closing queue' )
self.log.show( 'return results' )
return tmp
class AudioPlayer(object):
""" Play audio files.
"""
def __init__(self, filename: str='', show_position: bool=False, **kwargs):
""" AudioPlayer(filename='', show_position=False, **kwargs) -> Open
filename and an appropriate audio io for it.
"""
self._filename = filename
self._show_position = show_position
# Setup the msg_dict for sending messages to the child process.
self._msg_dict = Manager().dict()
self._control_dict = {}
# Create a pipe for sending and receiving messages.
self._control_conn, self._player_conn = Pipe()
# Open the file.
if filename:
self.open(filename, **kwargs)
def __str__(self) -> str:
""" The information about the open file.
"""
# Wait for the stream to open.
while 'info' not in self._msg_dict: pass
# Return the info string.
return self._msg_dict.get('info', '')
def __repr__(self) -> str:
""" __repr__ -> Returns a python expression to recreate this instance.
"""
repr_str = "filename='%(_filename)s'" % self.__dict__
return '%s(%s)' % (self.__class__.__name__, repr_str)
def __enter__(self):
""" Provides the ability to use pythons with statement.
"""
try:
return self
except Exception as err:
print(err)
return None
def __exit__(self, exc_type, exc_value, traceback):
""" Stop playback when finished.
"""
try:
self.stop()
self._control_conn.close()
self._player_conn.close()
return not bool(exc_type)
except Exception as err:
print(err)
return False
def __del__(self):
""" Stop playback before deleting.
"""
if self._control_dict.get('playing', False):
try:
self.stop()
except IOError:
pass
def __len__(self):
""" The length of the file if it has one.
"""
return self.length if self.length >= 0 else 0
def playing_wrapper(func):
""" Wrap methods and only call them if the stream is playing
"""
@functools_wraps(func)
def wrapper(self, *args, **kwargs):
""" Check if stream is playing and if it is then call func
otherwise print a message and exit.
"""
if not self.playing:
print("%(filename)s is not playing." % self._msg_dict)
return None
return func(self, *args, **kwargs)
return wrapper
def _play_proc(self, msg_dict: dict, pipe: Pipe):
""" Player process
"""
# Open the file to play.
try:
with open_file(cached=True, **msg_dict) as fileobj:
# Put the file info in msg_dict.
msg_dict['info'] = str(fileobj)
msg_dict['length'] = fileobj.length
if fileobj._rate < 44100:
# if py_imp == 'PyPy':
# blacklist = msg_dict.get('blacklist', [])
# blacklist.append('portaudio')
# msg_dict['blacklist'] = blacklist
# else:
import audioop
# msg_dict['rate'] = 44100
state = None
# Open an audio output device that can handle the data
# from fileobj.
device = open_device(fileobj, 'w', cached=True, **msg_dict)
try:
# Set the default number of loops to infinite.
fileobj.loops = msg_dict.get('loops', -1)
# Initialize variable.
buf = b'\x00' * device.buffer_size
written = 0
# Loop until stopped.
while msg_dict.get('playing', True):
# Stop if the read buffer is empty or player is
# not paused.
if not (buf or msg_dict.get('paused', False)):
break
# Print the stream position.
if msg_dict.get('show_position', False):
# Only print the position if the stream has a
# length.
if fileobj.length > 0:
# Calculate the percentage played.
pos = (fileobj.position * 100) / fileobj.length
# Make the string.
pos_str = 'Position: %.2f%%' % pos
# Find the length of the string.
format_len = len(pos_str) + 2
# Print the string and after erasing the old
# one using ansi escapes.
if py_imp == 'PyPy':
# Running in pypy which doesn't have the
# flush parameter in the print function.
print('\033[%dD\033[K%s' % (format_len,
pos_str), end='')
sys_stdout.flush()
else:
print('\033[%dD\033[K%s' % (format_len,
pos_str), end='', flush=True)
# Keep playing if not paused.
if not msg_dict.get('paused', False):
# Re-open the device after comming out of
# paused state.
if device.closed:
device = open_device(fileobj, 'w', cached=True,
**msg_dict)
# Read the next buffer full of data.
try:
buf = fileobj.readline()
except KeyboardInterrupt:
break
# if device._rate != fileobj._rate \
# and py_imp != 'PyPy' and fileobj._rate != 0:
if device._rate != fileobj._rate \
and fileobj._rate != 0:
# Convert the input sample rate to that of
# the output device.
buf, state = audioop.ratecv(buf,
fileobj._width,
fileobj._channels,
fileobj._rate,
int(device._rate),
state)
# Filler for end of partial buffer to elminiate
# end of audio noise.
if type(buf) == bytes:
filler = b'\x00' * (device.buffer_size - len(buf))
else:
filler = ''
# Write buf.
try:
written = device.write(buf + filler)
except KeyboardInterrupt:
break
else:
# Close the device when paused and sleep to
# open the audio for another process and
# save cpu cycles.
if not device.closed:
device.close()
time_sleep(0.05)
# Write a buffer of null bytes so the audio
# system can keep its buffer full.
# device.write(b'\x00' * device.buffer_size)
# Get and process any commands from the parent process.
if pipe.poll():
# Get the data into temp.
command = pipe.recv()
if 'getposition' in command:
pipe.send(fileobj.position)
elif 'setposition' in command:
fileobj.position = command['setposition']
elif 'getloops' in command:
pipe.send(fileobj.loops)
elif 'setloops' in command:
fileobj.loops = command['setloops']
elif 'getloopcount' in command:
pipe.send(fileobj.loop_count)
except Exception as err:
print(err)
finally:
if not device.closed:
device.close()
except IOError as err:
from time import sleep
msg_dict['error'] = err
msg_dict['info'] = ''
msg_dict['length'] = 0
print(err)
finally:
try:
# Set playing to False for the parent.
msg_dict['playing'] = False
except BrokenPipeError:
pass
def open(self, filename: str, **kwargs):
""" open(filename) -> Open an audio file to play.
"""
# Stop the current file from playing.
self.stop()
# Set the new filename.
self._filename = filename
# Reset the message dictionary so none of the old info is
# re-used.
self._msg_dict.clear()
# Fill the message dictionary with the new info.
self._msg_dict['show_position'] = self._show_position
self._msg_dict['filename'] = filename
self._msg_dict.update(kwargs)
self._control_dict.update(self._msg_dict)
# Pause it so when we call play later it will start the player
# but not the audio playback. Call play again to start audio
# playback.
self.pause()
# Start the playback process in a paused state. Requires a
# second call to play to un-pause.
self.play()
def play(self):
""" play() -> Start playback.
"""
if not self._msg_dict.get('playing', False):
# Set playing to True for the child process.
self._msg_dict['playing'] = True
# Open a new process to play a file in the background.
self._play_p = Process(target=self._play_proc,
args=(self._msg_dict, self._player_conn))
# Start the process.
self._play_p.start()
elif self._msg_dict.get('paused', True):
# Un-pause if paused.
self._msg_dict['paused'] = False
self._control_dict.update(self._msg_dict)
def stop(self):
""" stop() -> Stop playback.
"""
if self._msg_dict.get('playing', False):
# Stop playback.
self._msg_dict['playing'] = False
# Wait for the player process to stop.
self._play_p.join()
# Un-Pause.
self._msg_dict['paused'] = False
self._control_dict.update(self._msg_dict)
def pause(self):
""" pause() -> Pause playback.
"""
# Pause playback.
self._msg_dict['paused'] = True
self._control_dict.update(self._msg_dict)
@property
def error(self) -> bool:
""" True if playing.
"""
return self._msg_dict.get('error', False)
@property
def paused(self) -> bool:
""" True if playback is paused.
"""
return self._msg_dict.get('paused', False)
@property
def playing(self) -> bool:
""" True if playing.
"""
return self._msg_dict.get('playing', False)
@property
def length(self) -> int:
""" Length of audio.
"""
return self._msg_dict.get('length', 0)
@property
@playing_wrapper
def position(self) -> int:
""" Current position.
"""
self._control_conn.send('getposition')
return self._control_conn.recv()
@position.setter
@playing_wrapper
def position(self, value: int):
""" Set the current position.
"""
self._control_conn.send({'setposition': int(value)})
@property
@playing_wrapper
def loops(self) -> int:
""" Number of times to loop (playback time + 1).
"""
self._control_conn.send('getloops')
return self._control_conn.recv()
@loops.setter
@playing_wrapper
def loops(self, value: int):
""" Number of times to loop (playback time + 1).
"""
self._control_conn.send({'setloops': int(value)})
@property
@playing_wrapper
def loop_count(self) -> int:
""" Number of times the player has looped.
"""
self._control_conn.send('getloopcount')
return self._control_conn.recv()
@playing_wrapper
def seek(self, offset: int, whence=SEEK_SET) -> int:
""" seek(position) -> Seek to position in mod.
"""
if whence == SEEK_CUR:
self.position += offset
elif whence == SEEK_END:
self.position = self.length - offset
else:
self.position = offset
return self.position
@playing_wrapper
def tell(self) -> int:
""" tell -> Returns the current position.
"""
return self.position
class Queue_server(object):
'''
初始话公众号队列
@param Tuple wx_lists 公众号列表
'''
def __init__(self ,wx_lists=()):
self.__queue = Manager().Queue(-1)
self.init_wx_lists(wx_lists)
self.__fail_list = Manager().list()
'''
初始话公众号队列
@param Tuple wx_lists 公众号列表
'''
def init_wx_lists(self ,wx_lists=()):
for wx in wx_lists:
self.put(wx)
'''
添加元素
@param mixed value 要添加的元素
'''
def put(self ,value):
self.__queue.put(value)
'''
弹出元素
@return mixed
'''
def get(self):
if not self.empty():
return self.__queue.get()
return False
'''
获取队列
@return mixed
'''
def get_wx_lists_queue(self):
return self.__queue
'''
获取队列大小
@return int
'''
def get_size(self):
return self.__queue.qsize()
'''
队列是否为空
@return bool
'''
def empty(self):
return self.__queue.empty()
'''
添加失败数据
@param tuple wx_data 公众号信息
@return bool
'''
def put_fail_wx(self , wx_data):
self.__fail_list.append(wx_data)
'''
打印失败列表
'''
def print_fail_list(self ,flush=None):
if len(self.__fail_list) > 0 :
for fail in self.__fail_list:
self.put(fail)
print 'the fail wx : {0}' . format(fail)
if not flush:
self.__fail_list = Manager().list()
elif flush:
print 'all success'
#判断是否有错
def is_have_failed(self):
#判断是否有失败的公众号重新加入队列中
return not self.empty()
def get(self, key):
result_queue = Manager().Queue()
self.queue.put(('GET', (key,), result_queue,))
return result_queue.get()
class Media(object):
def __init__(self,movieFile):
self._movieFile = movieFile
self._isMovie = self._movieFile.endswith(".mp4") or self._movieFile.endswith(".ogg")
self._cond = mp.Condition()
self._queue = Manager().Queue(100)
self._totalFrames = mp.Value("i",0)
self._finished = mp.Value("i",0)
def _prefetch(self):
self._cond.acquire()
if self._isMovie:
self._cap = cv2.VideoCapture(self._movieFile)
else:
if not self._movieFile.endswith("/"):
self._movieFile = self._movieFile + "/"
self._imgFiles = glob.glob(self._movieFile + "*.jpg")
self._imgFiles.sort()
self._totalFrames.value = self._getTotalFrames()
self._cond.notify_all()
self._cond.release()
count = 0
while(self._finished.value == 0):
flag,frame = self._getNextFrame()
if flag :
try:
self._queue.put( frame , block=True )
count += 1
except:
pass
else:
break
self._closeMedia()
def openMedia(self):
self._cursor = 0
mp.Process(target=self._prefetch).start()
self._cond.acquire()
self._cond.wait()
self._cond.release()
def getTotalFrames(self):
return self._totalFrames.value
def _getTotalFrames(self):
if self._isMovie:
return int(self._cap.get(cv2.CAP_PROP_FRAME_COUNT))
else:
return len(self._imgFiles)
def _closeMedia(self):
if self._isMovie:
self._cap.release()
def getNextFrame(self):
if( self._cursor >= self._totalFrames.value):
return (False,None)
self._cursor += 1
return (True,self._queue.get())
def _getNextFrame(self):
if self._isMovie:
return self._cap.read()
else:
if( self._cursor < len(self._imgFiles )):
frame = cv2.imread(self._imgFiles[self._cursor])
self._cursor += 1
return (True,frame)
else:
return (False,None)
def closeMedia(self):
with self._finished.get_lock():
self._finished.value = 1
def search(output_dict, rules_file):
rules = [rule.split(' | ') for rule in pickle.load(open(rules_file, 'rb'))]
file_list = JoinableQueue()
word_dict = Manager().dict()
for root, subFolders, files in os.walk(os.path.join(os.path.dirname(__file__), 'corpus', 'tagged')):
for current_file in files:
if current_file.endswith(".pickle"):
file_list.put(os.path.join(root, current_file))
#break # TODO remove (only for testing with one file)
file_count = file_list.qsize()
def worker():
def rule_parser(tagged_data):
parser = nltk.RegexpParser('''
NP: {<NN|NNS|NNP|NNPS|NE>}
NPs: {<NP> (<,|CC> <NP>)+}
''')
return parser.parse(tagged_data)
def get_nltk_word(data):
if isinstance(data, nltk.tree.Tree):
if isinstance(data[0], tuple):
return data[0][0]
else:
return data[0]
else:
return data[0]
def add_to_dict(hypernym, hyponym):
if not hyponym in word_dict.keys():
old_list = word_dict.get(hypernym)
if not old_list:
old_list = [hyponym]
else:
if not hyponym in old_list:
old_list.append(hyponym)
word_dict[hypernym] = old_list
def apply_rules(data, position):
for rule in rules:
# search right side
if rule[0] == 'HYPERNYM':
possible_hypernym = get_nltk_word(data[position])
error = False
word_count = 1
for word in rule[1:-1]:
try:
if word != get_nltk_word(data[position + word_count]):
error = True
word_count += 1
except IndexError:
pass
try:
if not error:
if isinstance(data[position + word_count], nltk.tree.Tree):
if data[position + word_count].node == 'NP' and rule[-1] == 'NP':
add_to_dict(possible_hypernym, data[position + word_count][0][0])
break
elif data[position + word_count].node == 'NPs' and rule[-1] == 'NPs':
for node in data[position + word_count]:
if isinstance(node, nltk.tree.Tree):
add_to_dict(possible_hypernym, node[0][0])
break
except IndexError:
pass
# search left side
elif rule[-1] == 'HYPERNYM':
possible_hypernym = get_nltk_word(data[position])
error = False
word_count = -1
nrule = list(rule)
nrule.reverse()
for word in nrule[1:-1]:
try:
if word != get_nltk_word(data[position + word_count]):
error = False
word_count -= 1
except IndexError:
pass
try:
if not error:
if isinstance(data[position + word_count], nltk.tree.Tree):
if data[position + word_count].node == 'NP' and rule[-1] == 'NP':
add_to_dict(possible_hypernym, data[position + word_count][0][0])
break
elif data[position + word_count].node == 'NPs' and rule[-1] == 'NPs':
for node in data[position + word_count]:
if isinstance(node, nltk.tree.Tree):
add_to_dict(possible_hypernym, node[0][0])
break
except IndexError:
pass
while not file_list.empty():
input_file = file_list.get()
tagged_data = rule_parser(pickle.load(open(input_file, 'rb')))
for n in range(len(tagged_data)):
if isinstance(tagged_data[n], nltk.tree.Tree):
if tagged_data[n].node == 'NP':
apply_rules(tagged_data, n)
percentage = 100.0 - ((float(file_list.qsize()) / float(file_count)) * 100.0)
sys.stdout.write("\rProgress: {0:.2f}%".format(percentage))
sys.stdout.flush()
file_list.task_done()
sys.stdout.write("\rProgress: 0.00%")
for pid in range(8):
process = Process(target=worker, args=())
process.daemon = True
process.start()
file_list.join()
print('')
pickle_dict = dict()
for key in word_dict.keys():
pickle_dict[key] = word_dict.get(key)
pickle.dump(pickle_dict, open(output_dict, 'wb+'), 2)
