def easy_parallize(f, sequence):
from multiprocessing import Pool
#from multiprocessing.dummy import Pool
pool = Pool(processes=threads)
pool.map(f, sequence)
pool.close()
pool.join()
def start_crawlers(connector_class, num_processes=1):
"""
Starts a spider process for each spider class in the project
:param num_processes: the number of simultaneous crawling processes
:param connector_class: the connector class that should be used by the
spiders
"""
spider_classes = pyjobs_crawlers.tools.get_spiders_classes()
if num_processes == 0:
connector = connector_class()
with _get_lock('ALL') as acquired:
if acquired:
crawl(spider_classes, connector)
else:
print("Crawl process of 'ALL' already running")
return
# Splits the spider_classes list in x lists of size num_processes
spider_classes_chunks = list()
for x in range(0, len(spider_classes), num_processes):
spider_classes_chunks.append(spider_classes[x:x + num_processes])
# Start num_processes number of crawling processes
for spider_classes_chunk in spider_classes_chunks:
process_params_chunk = [(spider_class, connector_class)
for spider_class in spider_classes_chunk]
p = Pool(len(process_params_chunk))
p.map(start_crawl_process, process_params_chunk)
def loadExperiments():
(options, args) = _getArgs()
moduleName = args[0]
functionName = args[1]
paramsDir = args[2]
outputDir = args[3]
workers = options.workers
plotVerbosity = options.plotVerbosity
consoleVerbosity = options.consoleVerbosity
experimentModule = importlib.import_module(moduleName)
experimentFunction = getattr(experimentModule, functionName)
print "Loading parameter files from: {0}".format(paramsDir)
print "Output console and logs to: {0}\n".format(outputDir)
if not os.path.exists(outputDir):
os.makedirs(outputDir)
argsList = []
paramFileList = [f for f in listdir(paramsDir) if isfile(join(paramsDir, f))]
for f in paramFileList:
with open(join(paramsDir,f)) as paramFile:
params = yaml.safe_load(paramFile)
argsList.append((experimentFunction, params, paramsDir, f, paramFile,
outputDir, consoleVerbosity, plotVerbosity))
pool = Pool(processes=workers)
pool.map(runSingleExperiment, argsList)
def exclusivefs(train_x, train_y, test_x, test_y):
long = len(train_x[0])
findex = range(long)
splitnum = int(len(train_x)/2)
global tr_x
global tr_y
global te_x
global te_y
global y
tr_x = train_x[:splitnum]
tr_y = train_y[:splitnum]
te_x = train_x[splitnum:]
te_y = train_y[splitnum:]
y = [int(i) for i in te_y]
exclusivelist = []
pool = Pool(10)
print "exclusivelist"
exclusivelist = pool.map(iterindex,range(long,0,-1))
exclusivelistv2 = []
for i in exclusivelistv2:
for j in i:
exclusivelistv2.append(j)
print "exclusivemodel"
print "all %d runs" % len(exclusivelistv2)
pool = Pool(10)
pool.map(modelworker,exclusivelistv2)
def main(argv):
monthid = 0
try:
opts, args = getopt.getopt(argv, "m:", ['month'])
except getopt.GetoptError:
print 'mobilecheck.py -m <monthId>'
sys.exit(2)
for opt, arg in opts:
if opt in ('-m', '--month'):
monthid = arg
print 'MonthId [', monthid , ']'
if monthid != 0:
db = SpeedyDb();
print 'running mobile check'
sites = db.getUncheckedMobileSites(monthid)
print 'Processing ', len(sites), ' sites'
pool = Pool(processes=8)
pool.map(processSite, sites)
pool.close()
pool.join()
def main(argv):
#indicate start
print "Beginning download of every "+str(incrementCard)+'-th card'
print 'in the range' +str(startCard)+'-'+str(endCard)
#generate list of cards to spit out
#f = open('failuresTrue.txt', 'r')
#cardList = f.readlines()
cardList=[]
x = startCard
while x<=endCard:
cardList.append(x)
x+=incrementCard
#empty the failure file
f = open('FAILURES.txt','a')
f.truncate()
f.close()
#thread operation
p = Pool(numThreads)
p.map(callDownloadSinglePage, cardList)
print 'Done!'
def mutliprocess_start(process_num=20, limit=1000):
from multiprocessing import Pool as JPool # 多进程
from multiprocessing import cpu_count
pool = JPool(process_num * cpu_count())
pool.map(manual_start, (i for i in range(MAX_TASK_NUMBER / 5)))
pool.close()
pool.join()
def extract(self, parallel = True, metrics = True):
"""Extracts data from the netcdf files into a more useful format."""
self.pyhspf_names = ['rain',
'temperature',
'humidity',
'solar',
'snowdepth',
'evaporation',
'wind',
]
if parallel:
pool = Pool(len(self.pyhspf_names))
pool.map(self.organize, self.pyhspf_names)
pool.close()
else:
for v in pyhspf_names: self.organize(v)
# re-organize into NRCM classes at each grid point
stationdata = '{}/gridpoints'.format(self.destination)
if not os.path.isdir(stationdata): os.mkdir(stationdata)
if parallel:
pool = Pool(cpu_count())
x = pool.map(self.extract_gridpoints, self.names)
pool.close()
else:
for n in self.names: self.extract_gridpoints(n)
if metrics: self.extract_metrics()
def _do_multiprocess_clean(self):
params = []
for package_name in self._package_map:
params.append((self._npm, self._package_map[package_name]))
pool = Pool(cpu_count())
pool.map(clean_dependencies, params)
def start_crawlers(connector_class, processes=1, debug=False):
"""
Start spider processes
:param processes:
:param connector_class:
:param debug:
:return:
"""
spiders_classes = get_spiders_classes()
if processes == 0:
connector = connector_class()
with _get_lock('ALL') as acquired:
if acquired:
crawl(spiders_classes, connector, debug)
else:
print("Crawl process of 'ALL' already running")
return
# split list in x list of processes count elements
spider_classes_chunks = [spiders_classes[x:x + processes] for x in range(0, len(spiders_classes), processes)]
# Start one cycle of processes by chunk
for spider_classes_chunk in spider_classes_chunks:
process_params_chunk = [(spider_class, connector_class, debug) for spider_class in spider_classes_chunk]
p = Pool(len(process_params_chunk))
p.map(start_crawl_process, process_params_chunk)
def main():
from multiprocessing import Pool
p = Pool(20)
begin = time.perf_counter()
p.map(run_client, [100] * 5)
end = time.perf_counter()
print((end - begin) * 1000)
def main():
"""Take in the arguments"""
parser = argparse.ArgumentParser(description='Run the Benchmark script')
parser.add_argument('host', action="store", help='HostName')
parser.add_argument('number_iterations', action="store", type=int, help='Number of iterations')
parser.add_argument('number_processes', action="store", type=int, help='Number of processes')
result = parser.parse_args()
global SITE_HOST
SITE_HOST = result.host
zidx_list = range(result.number_iterations)
random.shuffle(zidx_list)
post_list = []
for zidx in zidx_list:
post_list.append(generateURL(zidx))
from multiprocessing import Pool
pool = Pool(result.number_processes)
start = time.time()
pool.map(postURLHelper, post_list)
print time.time() - start
# KL TODO insert get data here
getURL(generateURL2(zidx_list[0]))
getURL(generateURL2(zidx_list[0]))
def test_runMediaTest(self,server,user,password):
#local use
#/Users/mgarthwaite/Dropbox/CAH_Recorded
#/zoidberg/CI/CAH_Recorded
pictureList, videoList = self.appendList("/zoidberg/CI/CAH_Recorded")
if len(pictureList) > len(videoList):
listCount = len(pictureList)
else:
listCount = len(videoList)
func = partial(runTest, server, user, password, pictureList, videoList)
pool = Pool(processes=8)
pool.map(func,range(0,listCount))
pool.close()
pool.join()
logFile = self.aggregateLogs(listCount)
logFile.seek(0)
data = logFile.read()
count = data.count("fail: 1")
if count == 0:
print "NO FAILED TESTS. GREEN"
sys.exit(0)
elif count < listCount and count > 0:
print "SOME FAILED TESTS. YELLOW"
sys.exit(0)
else: #count >= listCount
print "ALL TESTS FAILED. RED."
sys.exit(1)
def main():
start()
for url in seen_urls:
print(url)
# print(seen_urls)
p = Pool(4)
p.map(get_data, next_urls)
def runCurve(self, runAnalysis=True, delete = False):
tbins = np.arange(float(self.curveConf['tstart']),
float(self.curveConf['tstop']),
float(self.curveConf['tstep']))
bins = np.arange(0,np.size(tbins))
binsinfo = zip(np.arange(0,np.size(tbins)),
tbins,
[self.commonConf for bin in bins],
[self.analysisConf for bin in bins],
[self.curveConf for bin in bins])
if(runAnalysis):
if int(self.commonConf['multicore']) > 1:
pool = Pool(processes = int(self.commonConf['multicore']))
pool.map(runAnalysisStepMP,binsinfo)
else:
for bininfo in binsinfo:
runAnalysisStepMP(bininfo)
if(delete):
templist = glob.glob("*_bin" + str(binnum) + "*")
for t in templist:
os.remove(t)
def fetch_imagery(image_locations, local_dir):
pool = Pool(cpu_count())
tupled = [(loc[0], loc[1], local_dir) for loc in image_locations]
try:
pool.map(fetch_imagery_uncurried, tupled)
finally:
pool.close()
def mass_tri_plot(data, savedir, name='plot', Type='speed', Map=False):
"""
Plots all time series. Makes use of multiprocessing for speed.
"""
trigrid = data['trigrid']
#get the data to plot
try:
toPlot = data[Type]
except KeyError:
print Type + " is not an element of data. Please calculate it."
raise Exception("Invalid dictionary entry")
#set the variable as a global variable
global plotvar
plotvar = toPlot
global saveDir
saveDir = savedir
global grid
grid = trigrid
#see if the save directory exists, or make it
if not os.path.exists(savedir):
os.makedirs(savedir)
l = toPlot.shape[0]
p = Pool(4)
plt.gca().set_aspect('equal')
p.map(save_plot, range(50))
clearall()
def do_experiment(self, params):
""" runs one experiment programatically and returns.
params: either parameter dictionary (for one single experiment) or a list of parameter
dictionaries (for several experiments).
"""
paramlist = self.expand_param_list(params)
# create directories, write config files
for pl in paramlist:
# check for required param keys
if ('name' in pl) and ('iterations' in pl) and ('repetitions' in pl) and ('path' in pl):
self.create_dir(pl, self.options.delete)
else:
print 'Error: parameter set does not contain all required keys: name, iterations, repetitions, path'
return False
# create experiment list
explist = []
# expand paramlist for all repetitions and add self and rep number
for p in paramlist:
explist.extend(zip( [self]*p['repetitions'], [p]*p['repetitions'], xrange(p['repetitions']) ))
# if only 1 process is required call each experiment seperately (no worker pool)
if self.options.ncores == 1:
for e in explist:
mp_runrep(e)
else:
# create worker processes
pool = Pool(processes=self.options.ncores)
pool.map(mp_runrep, explist)
return True
def test_word2id():
"""把测试集的所有词转成对应的id。"""
time0 = time.time()
print('Processing eval data.')
df_eval = pd.read_csv('../raw_data/question_eval_set.txt', sep='\t', usecols=[0, 2, 4],
names=['question_id', 'word_title', 'word_content'], dtype={'question_id': object})
print('test question number %d' % len(df_eval))
# 没有 title 的问题用 content 来替换
na_title_indexs = list()
for i in xrange(len(df_eval)):
word_title = df_eval.word_title.values[i]
if type(word_title) is float:
na_title_indexs.append(i)
print('There are %d test questions without title.' % len(na_title_indexs))
for na_index in na_title_indexs:
df_eval.at[na_index, 'word_title'] = df_eval.at[na_index, 'word_content']
# 没有 content 的问题用 title 来替换
na_content_indexs = list()
for i in tqdm(xrange(len(df_eval))):
word_content = df_eval.word_content.values[i]
if type(word_content) is float:
na_content_indexs.append(i)
print('There are %d test questions without content.' % len(na_content_indexs))
for na_index in tqdm(na_content_indexs):
df_eval.at[na_index, 'word_content'] = df_eval.at[na_index, 'word_title']
# 转为 id 形式
p = Pool()
eval_title = np.asarray(p.map(get_id4words, df_eval.word_title.values))
np.save('../data/wd_eval_title.npy', eval_title)
eval_content = np.asarray(p.map(get_id4words, df_eval.word_content.values))
np.save('../data/wd_eval_content.npy', eval_content)
p.close()
p.join()
print('Finished changing the eval words to ids. Costed time %g s' % (time.time() - time0))
def train_word2id():
"""把训练集的所有词转成对应的id。"""
time0 = time.time()
print('Processing train data.')
df_train = pd.read_csv('../raw_data/question_train_set.txt', sep='\t', usecols=[0, 2, 4],
names=['question_id', 'word_title', 'word_content'], dtype={'question_id': object})
print('training question number %d ' % len(df_train))
# 没有 content 的问题用 title 来替换
na_content_indexs = list()
for i in tqdm(xrange(len(df_train))):
word_content = df_train.word_content.values[i]
if type(word_content) is float:
na_content_indexs.append(i)
print('There are %d train questions without content.' % len(na_content_indexs))
for na_index in tqdm(na_content_indexs):
df_train.at[na_index, 'word_content'] = df_train.at[na_index, 'word_title']
# 没有 title 的问题， 丢弃
na_title_indexs = list()
for i in xrange(len(df_train)):
word_title = df_train.word_title.values[i]
if type(word_title) is float:
na_title_indexs.append(i)
print('There are %d train questions without title.' % len(na_title_indexs))
df_train = df_train.drop(na_title_indexs)
print('After dropping, training question number(should be 2999952) = %d' % len(df_train))
# 转为 id 形式
p = Pool()
train_title = np.asarray(p.map(get_id4words, df_train.word_title.values))
np.save('../data/wd_train_title.npy', train_title)
train_content = np.asarray(p.map(get_id4words, df_train.word_content.values))
np.save('../data/wd_train_content.npy', train_content)
p.close()
p.join()
print('Finished changing the training words to ids. Costed time %g s' % (time.time() - time0))
def forum_scrape(html):
#get response
response = get_request(html)
#get soup object
soup = soupify(response)
#get list of threads and links
threads = parse_thread_page(soup)
#add all threads and sub-threads to the database w/ multiprocessing
p = Pool()
p.map(add_to_db, threads)
#while there is a next button
if soup.find("a", rel="next") != None:
next_page = soup.find("a", rel="next")
page = 1
print("page:", page)
while next_page:
href = next_page['href']
response = get_request(href)
page +=1
print("----")
print("page:", page)
soup = soupify(response)
threads = parse_thread_page(soup)
p = Pool()
p.map(add_to_db,threads)
next_page = soup.find("a", {"rel": "next"})
def updateTranslation(args):
# Get map that contains (besides other stuff)
# the crowdin ID for a given file
translationFilemap = getTranslationFilemapCache(args.language, args.force_filemap_update)
# Collect valid downloadable files for parallel processing
fileinfos = []
for filename, fileinfo in translationFilemap.items():
filepath = os.path.join("cache", args.language, fileinfo["path"])
# Create dir if not exists
try: os.makedirs(os.path.dirname(filepath))
except OSError as exc:
if exc.errno == errno.EEXIST:
pass
else:
raise
fileid = fileinfo["id"]
fileinfos.append((fileid, filepath))
# Curry the function with the language
performDownload = functools.partial(performPOTDownload, args.language)
# Perform parallel download
if args.num_processes > 1:
pool = Pool(args.num_processes)
pool.map(performDownload, fileinfos)
else:
for t in fileinfos:
performDownload(t)
#Set download timestamp
timestamp = datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
with open("lastdownload.txt", "w") as outfile:
outfile.write(timestamp)
def train_bolt_classifier_batch(
dataset_dir="",
output_dir="",
modeltype="sgd_maxent_l2",
verbset_path="",
selftest=False,
cls_option={"loss": "log", "epochs": 10, "alpha": 0.0001, "reg": "L2"},
pool_num=2,
):
vs_file = pickle.load(open(verbset_path, "rb"))
verbs = vs_file.keys()
verbsets = deepcopy(vs_file)
set_names = [os.path.join(dataset_dir, v) for v in verbs]
po = Pool(processes=pool_num, maxtasksperchild=16)
args = []
for idd, dir in enumerate(set_names):
dspath = os.path.join(dir)
outputpath = os.path.join(output_dir, dir)
if not os.path.exists(outputpath):
os.makedirs(outputpath)
arg = {"dataset_dir": dspath, "output_path": outputpath, "modeltype": modeltype, "options": cls_option}
args.append(arg)
po.map(train_bolt_classifier_p, args)
po.close()
po.join()
def lvl2_data(obsid_list):
p = Pool(4)
if len(glob.glob(obs+'/*/*evt2.fits')) == 1:
print('Found obsID repro directory')
else:
p.map(repro_worker, obsid_list)
def main():
global V_SHAPE, N_PROCS, DST
if not len(sys.argv) in [1,2,3,4]: exit()
subset = "train"
if len(sys.argv) > 1: V_SHAPE = (int(sys.argv[1]),)*2
if len(sys.argv) > 2: N_PROCS = int(sys.argv[2])
if len(sys.argv) > 3: subset = int(sys.argv[3])
if not subset in ["train", "test"]: exit()
DST = "data/gray_i%i_%s/"%(V_SHAPE[0], subset)
print "size", V_SHAPE, "jobs", N_PROCS, "subset", subset
# cleanup
to_delete = glob("data/%s/Sample*/"%subset) # folders
for p in to_delete: shutil.rmtree(p)
to_delete = glob("/dev/shm/Sample*.mp4")
for p in to_delete: os.remove(p)
# grab sample files
file_paths = glob("data/%s/*.zip"%subset)
file_paths.sort()
print len(file_paths), "files found"
assert len(file_paths) > 0
# make sure no file is missing
offset = 1 if not subset == "test" else 701
for i, path in enumerate(file_paths):
assert int(path.split("/")[-1][-8:-4]) == i + offset
if not os.path.exists(DST): os.makedirs(DST)
# preprocess
pool = Pool(N_PROCS)
# file_paths.sort(revert=True)
pool.map(partial(extract_depth, subset=subset), file_paths)
def directoryProcessor(dir, parallel):
files = os.listdir(dir)
p = Pool(parallel)
tasks = []
for i in range(len(files)):
tasks.append((J(dir, files[i]), i))
p.map(fileProcessor, tasks)
def main(argv):
# help option
try:
opts, args = getopt.getopt(argv, 'h')
for opt, arg in opts:
if opt == '-h':
print('''args: 3 pickle files folder_of_data number_of_cores:
clf_alc.p clf_fpa.p clf_fpl.p tweets_folder 8''')
sys.exit(2)
except getopt.GetoptError:
print('-h for help')
sys.exit(2)
# run prediction
s_clf_alc, s_clf_fpa, s_clf_fpl, folder, out_dir, cores = tuple(argv)
cores = int(cores)
clf_alc = pickle.load(open(s_clf_alc, 'rb'))
clf_fpa = pickle.load(open(s_clf_fpa, 'rb'))
clf_fpl = pickle.load(open(s_clf_fpl, 'rb'))
clf = PredictionTransformer(clf_alc, clf_fpa, clf_fpl)
# parallel
p = Pool(cores)
dirs = [(clf, out_dir, folder + '/' + f) for f in os.listdir(folder)]
p.map(predict, dirs)
def evaluate(self, **args):
""" Evaluate all individuals in population, calls the evaluate() method of individuals
:param args: this params are passed to the evaluation function
"""
# We have multiprocessing
if self.multiProcessing[0] and MULTI_PROCESSING:
logging.debug("Evaluating the population using the multiprocessing method")
proc_pool = Pool()
# Multiprocessing full_copy parameter
if self.multiProcessing[1]:
results = proc_pool.map(multiprocessing_eval_full, self.internalPop)
proc_pool.close()
proc_pool.join()
for i in xrange(len(self.internalPop)):
self.internalPop[i] = results[i]
else:
results = proc_pool.map(multiprocessing_eval, self.internalPop)
proc_pool.close()
proc_pool.join()
for individual, score in zip(self.internalPop, results):
individual.score = score
else:
for ind in self.internalPop:
ind.evaluate(**args)
self.clearFlags()
def reduce(self, params):
from os import listdir
files = [f for f in listdir(".")
if f.startswith(params.prefix) and f.endswith(".root")]
def get_period(s): return s.split("-")[1].split("_")[-1]
by_subperiod = {}
for f in files:
by_subperiod.setdefault(get_period(f), []).append(f)
from pprint import pprint
pprint(by_subperiod)
from multiprocessing import Pool, cpu_count
pool = Pool(cpu_count())
pool.map(mp_merge, [("period%s.root" % p, files) for p, files in by_subperiod.iteritems()])
by_period = {}
for p, files in sorted(by_subperiod.iteritems()):
by_period.setdefault(p[0], []).append("period%s.root" % p)
pprint(by_period)
pool.map(mp_merge, [("period%s.root" % p, files) for p, files in by_period.iteritems()])
from hmerge import merge_files
merge_files("all.root", ["period%s.root" % p for p in by_period])
print "Done."
def main():
if not os.path.isfile(task_filename):
print 'Tasks not exist.'
return None
tasks = [task.strip('\r').strip('\n').strip() for task in open(task_filename, 'r').readlines()]
dones = []
if os.path.isfile(done_filename):
dones = [task.strip('\r').strip('\n').strip() for task in open(done_filename, 'r').readlines()]
for task in tasks:
if not task in dones:
task_queue.put(task)
time.sleep(1)
if not task_queue.empty():
pool = Pool(thread + 1)
try:
pool.map(process, range(thread + 1))
pool.close()
pool.join()
print '\n*************************************'
print '* Tasks all done *'
print '*************************************'
except:
print '\n*************************************'
print '* Tasks be terminated *'
print '*************************************'
finally:
task_queue.cancel_join_thread()
done_queue.cancel_join_thread()
result_queue.cancel_join_thread()
pool.terminate()
split = None
cat_names = get_cat_names(config.data)
write_cat_names(config.data, config.output)
input_categories = sorted(
[os.path.join(config.data, cat) for cat in cat_names])
output_categories = sorted(
[os.path.join(config.output, cat) for cat in cat_names])
all_test_file = os.path.join(config.output, 'test.txt')
all_train_file = os.path.join(config.output, 'train.txt')
arguments = [(input_categories[i], output_categories[i], i, all_test_file,
all_train_file, config.dataset_type, config.adaptive,
config.num_rotations, split)
for i in range(0, len(input_categories))]
pool = Pool(processes=config.num_threads)
pool.map(convert_one_category, arguments)
pool.close()
pool.join()
print('started with lmdb builidng')
os.system(
'/opt/caffe/build/tools/convert_octree_data {} \"{}\" \"{}\"'.format(
"/", os.path.join(config.output, 'test.txt'),
os.path.join(config.output, 'test_lmdb')))
os.system(
'/opt/caffe/build/tools/convert_octree_data {} \"{}\" \"{}\"'.format(
"/", os.path.join(config.output, 'train.txt'),
os.path.join(config.output, 'train_lmdb')))
# PEMROSESAN PARALEL DENGAN multiprocessing.process
print("\nPemrosesan Paralel dengan multiprocessing.process")
kumpulan_proses = []
process_awal = time()
for i in range(angkaawal, angkaakhir + 1):
p = Process(target=gan_gen1, args=(i, ))
kumpulan_proses.append(p)
p.start()
for i in kumpulan_proses:
p.join()
process_akhir = time()
# PEMROSESAN PARALEL DENGAN multiprocessing.pool
print("\nPemrosesan Paralel dengan multiprocessing.pool")
pool_awal = time()
pool = Pool()
pool.map(gan_gen, range(angkaawal, angkaakhir + 1))
pool.close()
pool_akhir = time()
print("\nSekuensial", sekuensial_akhir - sekuensial_awal, "detik")
print("multiprocessing.process", process_akhir - process_awal, "detik")
print("multiprocessing.pool", pool_akhir - pool_awal, "detik")
# crop gt
gt_np = np.asarray(gt, dtype='uint8')
gt_crop_np = gt_np[hei_top:hei_bot, wid_lef:wid_rig]
gt_crop = Image.fromarray(gt_crop_np, mode='L')
gt_crop = gt_crop.resize((224, 224))
gt_crop.save(res_gt + mask_name)
# # crop image mask, resize to 224x224 and save
# mask_crop_np = mask_np[hei_top:hei_bot, wid_lef:wid_rig]
# mask_crop = Image.fromarray(mask_crop_np, mode='L')
# mask_crop = mask_crop.resize((224, 224))
# mask_crop.save(res_fomask + mask_name)
# image = Image.open(ori_img_rp + img_name)
# image_np = np.asarray(image, dtype='uint8')
# image_crop_np = image_np[hei_top:hei_bot, wid_lef:wid_rig, :]
# image_crop = Image.fromarray(image_crop_np, mode='RGB')
# image_crop = image_crop.resize((224, 224))
# image_crop.save(res_image + mask_name)
if __name__ == '__main__':
num_processor = 60
fol_set = os.listdir(fol_sys_mask_rp)
fol_set.sort()
pool = Pool(num_processor)
run = pool.map(prepare_data, fol_set)
pool.close()
pool.join()
import os
import glob
import cv2
from multiprocessing import Pool
def get_frame(files):
for file in files:
filedir=file.rsplit('.mp4',1)[0]
filedir='pic/'+filedir
if not os.path.exists(filedir):
os.mkdir(filedir)
print(file)
cap=cv2.VideoCapture(file)
fid=0
success=1
while(success):
success,frame=cap.read()
picname=filedir+'/'+str(fid)+'.jpg'
cv2.imwrite(picname,frame)
fid+=1
cap.release()
if __name__=="__main__":
train_files=glob.glob('./train/*/*.mp4')
val_files=glob.glob('./val/*/*.mp4')
l1=len(train_files)
l2=len(val_files)
pool=Pool(6)
f=[train_files[0:l1/6],train_files[l1/6,l1/3],train_files[l1/3,l1/2],train_files[l1/2,l1*2/3],train_files[l1*2/3,l1],val_files]
pool.map(get_frame,f)
pool.close()
pool.join()
