def filecz():
try:
f = open(r'')
print(f.read())
finally:
if f:
f.close()
with open(r'', 'r') as filereader:
print(filereader.read())
for line in filereader.readlines():
print(line.strip())
print(os.getpid())
if __name__ == '__main__':
print(os.getpid())
for i in range(5):
p = process(target=filecz, args=(str(i), ))
p.start()
p.join()
time.sleep(random.random() * 3)
if __name__ == '__main__':
p = pool(process=3)
for i in range(5):
p.apply_async(filecz, args=(i, ))
p.close()
p.join()
def cpmap(cores=1):
"""Configurable parallel map context manager.
Returns appropriate map compatible function based on configuration:
- Local single core (the default)
- Multiple local cores
"""
if int(cores) == 1:
yield itertools.imap
else:
if multiprocessing is None:
raise ImportError("multiprocessing not available")
# Fix to allow keyboard interrupts in multiprocessing: https://gist.github.com/626518
def wrapper(func):
def wrap(self, timeout=None):
return func(self, timeout=timeout if timeout is not None else 1e100)
return wrap
IMapIterator.next = wrapper(IMapIterator.next)
# recycle threads on Python 2.7; remain compatible with Python 2.6
try:
pool = multiprocessing.pool(int(cores), maxtasksperchild=None)
except TypeError:
pool = multiprocessing.Pool(int(cores))
yield pool.imap_unordered
pool.terminate()
def save_model_picture_multiprocessing(train_list, train_numpy):
'''
too slow, use pool in multiprocessing
save GMM model for every picture, for we have split the train and test set, do it split
:param train_list:
:param train_numpy:
:return: every picture model
'''
picture_list = []
save_path_list = []
start_time = time.time()
for i, train_path in enumerate(train_list):
picture = train_numpy[i]
# train_path:1000/1000 so we should create directory 1000 first
save_dir = 'model_picture/' + train_path.split('/')[0]
if not os.path.exists(save_dir):
os.mkdir(save_dir)
save_path = 'model_picture/' + train_path + '.model'
save_path_list.append(save_path)
picture_list.append(picture)
picture_save_path = zip(picture_list, save_path_list)
p = pool(32)
p.map(save_model_picture_for_multipro, picture_save_path)
p.close()
p.join()
end_time = time.time()
print('time for 10 picture is {}'.format(end_time - start_time))
def set_pool(self, name: str, *args, **kwargs):
"""
создать пул
"""
pool = self.pools[name]
pool = pool(*args, **kwargs)
self.pool = pool
return pool
def main():
how_many = 50
p = pool(processes=how_many)
parse_us = [random_starting_url() for _ in range(how_many)]
data = p.map(get_links [link for link in parse_us])
data = [url for url_list in data for url in url_list]
p.close()
with open('D:\\intermediate\\urls.txt', 'w') as f:
f.write(str(data))
def thread_assign(self):
mobile = 130
ls: list = []
for i in range(10):
ls.append(mobile + i)
p = pool(processes=self.assign_num)
ls1 = p.map(self.num_assign, ls)
p.close()
p.join()
print(ls1)
def __init__(self):
indices = []
for i in range(1000):
for j in range(1000):
for k in range(1000):
indices.append((i, j, k))
start_time = time.time()
p = multiprocessing.pool(multiprocessing.cpu_count())
p.map(self.calcaulte, indices)
p.close()
p.join()
print('time consumed is {}'.format(time.time() - start_time))
def proc_data(self, func, tt):
print tt
po = pool(self.processor_count)#mp.cpu_count()
multiple_results = [po.apply_async(func, (self.divData[i],self.resData[i])) for i in range(self.processor_count)]
#[res.get(timeout=None) for res in multiple_results]
for res in multiple_results:
res.get(timeout=None)
#p = pro(target=func, args=(self.divData[0], self.resultData[0]))
#p.start()
#p.join()
pass
def Gradual_Parallel_MST(ss, idx, Scenario_Outcomes, mst):
subsets = []
subset1 = [
s for s in ss if str(scenario_outcomes[s][idx]) in list(cvar[1])
]
subset2 = [
s for s in ss if str(scenario_outcomes[s][idx]) not in list(cvar[1])
]
if subset2 != []:
new_sets.append(subset2)
subsets.append(subset2)
if len(list(cvar[1])) > 1:
sortable = [(s, scenario_outcomes[s][idx]) for s in subset1]
subset3 = groupby_func(sortable, key=itemgetter(1))
if subset3 != []:
new_sets.append(subset3)
subsets.append(subset3)
else:
if subset1 != []:
new_sets.append(subset1)
subsets.append(subset1)
### generate multi-process object and run mst subset gen
if len(sets) < mp.cpu_count():
np = len(sets)
else:
np = mp.cpu_count()
### create pool
pool = mp.pool(np)
### run pool
results = [
pool.apply_async(subset_mst, args=(ss, scenario_outcomes, mst))
for ss in subsets
]
pool.close()
pool.join()
NAC_to_add = {}
NAC_to_add = NAC_to_add.union(
*[results[i]._value for i in range(len(results))])
return NAC_to_add, subsets
def main():
path = "/home/pi/data/{}".format(uuid.uuid4())
os.mkdir(path)
p = pool(20)
i2cq = Queue(maxsize=200)
gpsq = Queue(maxsize=4)
p.apply_async(readGPS, gpsq)
p.apply_async(readI2c, i2cq)
time.sleep(200)
while True:
data = list()
for i in range(0, 500):
currData = makeCurrData(i2cq, gpsq)
data.append(currData)
print(data)
def get_img(self, num_threads=None, img=True, dcm=True):
print("Extracting images from LUNA16 Dataset")
for subdir in self.subsetdirs:
src_dir = os.path.join(self.data_path, subdir)
src_paths = glob(src_dir + '/*.mhd')
if num_threads:
pool = multiprocessing.pool(num_threads)
pool.map(self.process_image, src_paths)
else:
for src_path in src_paths:
print("src Path: ", src_path)
img_array, patient_id = self.process_image(src_path)
if self.img:
#print(self.img_path)
#dst_dir = os.path.join(self.img_path,str(patient_id))
dst_dir = self.img_path + '/' + str(patient_id) + '/'
self.writetoimg(img_array, dst_dir)
if self.mhd:
dst_dir = self.mhd_path + '/' + str(patient_id)
self.writetomhd(img_array, dst_dir)
def run_iteration():
start = time.time()
# 超时设置
# 这两种设置都进行才有效。如果超时，会报错（若不设置，则超时多久都不报错）
document.set_page_load_timeout(5)
document.set_script_timeout(5)
p = pool()
global bank_info
bank_info = p.map(itr_banks, range(len(bank_sites)))
# flatten the list using sum
bank_info = sum(bank_info, [])
p.close()
p.join()
end = time.time()
print("Total time used on fetching: %0.3f" % (end - start))
# a new manipulation will be needed if new bank's exchange rates are added to bank_info
for e in bank_info: # normalize
e[0] = re.sub("\(.*\)", "", e[0])
e[1] = re.sub("\(.*\)", "", e[1])
e[0] = re.sub("/.*", "", e[0])
e[1] = re.sub("/.*", "", e[1])
e[3] = 1.0 / e[3]
# !!!!!
# CAUTION THERE MAY BE LOSSES OF WORD 现钞，现汇
for item in bank_info:
name0 = item[0]
name1 = item[1]
if name0 not in currencies:
currencies.append(name0)
if name1 not in currencies:
currencies.append(name1)
def main():
client = InfluxDBClient(host=myhost,
port=myport,
username=myuser,
password=mypass,
ssl=True,
verify_ssl=True)
databases = client.get_list_database()
for every_db in databases:
container[every_db['name']] = {} # assign database name - 1-st level;
client.switch_database(every_db['name'])
try:
print(f'Get measurements for [{every_db}] database')
measurements = client.get_list_measurements()
# print(measurements)
except Exception as e:
print(
f'ERROR for database [{every_db}] of get measurements as: [{e}]'
)
continue
for every_meas in measurements:
container[every_db['name']][every_meas['name']] = {
} # assign measure name - 2-nd level;
# create temporary working object to obtain fields:
working_list = []
for each_db in container:
for each_meas in container[each_db]:
temp_object = {
'host': myhost,
'port': myport,
'user': myuser,
'pass': mypass,
'database': each_db,
'measure': each_meas
}
working_list.append(temp_object)
############################################################################
# get all fields of influx and put them into temporary storage
with pool(
len(working_list)
if len(working_list) < maxconnections else maxconnections) as p:
get_fields_list = p.map(call_fields, working_list)
get_tags_list = p.map(call_tags, working_list)
del p # remove pool of workers
# move info about fields into main storage variable
for each_record in get_fields_list:
db_name = list(each_record.keys())[0]
# print("f> "+db_name) # debug
meas_name = list(each_record.get(db_name))[0]
# print("f>> "+meas_name) # debug
container[db_name][meas_name]['fields_list'] = [
] # set empty array for received fields
container[db_name][meas_name]['fields'] = {
} # set empty dict for store info separately
try:
for field in each_record[db_name][meas_name]:
field_name = field[0]
# print("f>>> "+field_name) # debug
container[db_name][meas_name]['fields_list'].append(field[0])
container[db_name][meas_name]['fields'][field_name] = {}
container[db_name][meas_name]['fields'][field_name][
'name'] = field[0]
container[db_name][meas_name]['fields'][field_name][
'type'] = field[1]
except Exception as e:
print(
f'ERROR - catch during field iteration in working_list for field [{field}]'
)
print(e)
continue
del get_fields_list # clean temporary storage;
# move info about tags into main storage variable
for each_record in get_tags_list:
db_name = list(each_record.keys())[0]
# print("t> "+db_name) # debug
meas_name = list(each_record.get(db_name))[0]
# print("t>> "+meas_name) # debug
container[db_name][meas_name]['tags_list'] = [
] # set empty array for received tags
container[db_name][meas_name]['tags'] = {
} # set empty dict for store info separately
try:
for tag in each_record[db_name][meas_name]:
tag_name = tag[0]
# print("t>>> "+tag_name) # debug
container[db_name][meas_name]['tags_list'].append(tag_name)
container[db_name][meas_name]['tags'][tag_name] = {}
container[db_name][meas_name]['tags'][tag_name]['name'] = tag[
0]
except Exception as e:
print(
f'ERROR - catch during tags iteration in working_list for tag [{tag}]'
)
print(e)
continue
del get_tags_list # clean temporary storage;
# fill temporary storage for retrive last record to fields
working_list = []
for each_db in container:
# print(">"+str(container[each_db])) # debug
for each_meas in container[each_db]:
# print(">>"+str(container[each_db][each_meas])) # debug
# print(each_db + " > " + each_meas)
# print(container[each_db][each_meas])
for each_field in container[each_db][each_meas]['fields_list']:
# print('## each_ms = '+str(each_ms))
# print(">>>"+str(container[each_db][each_meas]['fields_list'])) # debug
temp_object = {
'host': myhost,
'port': myport,
'user': myuser,
'pass': mypass,
'database': each_db,
'measure': each_meas,
'field': each_field
}
working_list.append(temp_object)
# get info about last record to every field time/value:
with pool(
len(working_list)
if len(working_list) < maxconnections else maxconnections) as p:
get_last_records = p.map(call_query, working_list)
# move received info into main storage:
for each_record in get_last_records:
db_name = list(each_record.keys())[0]
meas_name = list(each_record.get(db_name))[0]
# print(db_name) # debug
# print(meas_name) # debug
try:
for field in each_record[db_name][meas_name]:
# print(field) # debug
field_last_time = each_record[db_name][meas_name][field][
'time']
field_last_value = each_record[db_name][meas_name][field][
'value']
# move it to CONTAINER
container[db_name][meas_name]['fields'][field][
'last_time'] = field_last_time
container[db_name][meas_name]['fields'][field][
'last_value'] = field_last_value
except Exception as e:
print(
f'ERROR - parsing working_list for last time record = field [{field}]'
)
print(e)
continue
del get_last_records # clean temporary storage
# create and fill html report of all this stuff:
report = open(report_file, 'w')
report.write(
html_startpart) # write first part for all css style and basic stuff;
for db in container:
print(db)
for meas in container[db]:
print(meas)
report.write('<div class="row">')
report.write(f'<div class="data_db"><span>{db}</span></div>'
) # end of database block (1-st column);
report.write(
f'<div class="data_meas"><span>{meas}</span></div><div class="data_tags">'
) # end of measurement block (2-nd column)
try:
for tag in container[db][meas]['tags_list']:
report.write(f'<span>{tag}</span>')
except Exception as e:
print(
f'ERROR - Processing [{meas}] in [{db}] and try iterate over ["tags_list"] and catch [{e}]'
)
report.write('</div><div class="data_fields">'
) # end of tags block (3-rd column);
try:
for field in container[db][meas]['fields_list']:
report.write(f'<span>{field}</span>')
except Exception as e:
print(
f'Processing [{meas}] in [{db}] and try iterate over ["fields_list"] and catch [{e}]'
)
report.write('</div><div class="field_last_record">'
) # end for fields block (4-th column);
# report.write('</div>') # end for fields block (4-th column);
try:
for field in container[db][meas]['fields_list']:
last_time = container[db][meas]['fields'][field][
'last_time']
last_value = container[db][meas]['fields'][field][
'last_value']
# report.write(f'<span>[{field}]=>[{last_time}]=[{last_value}]</span>')
report.write(f'<span>[{last_time}]</span>')
except Exception as e:
print(
f'Processing [{meas}] in [{db}] and try iterate over ["fields_list"] to put LAST time/value and catch [{e}]'
)
report.write('</div>') # end of last record block
report.write('</div>') # end for row block
report.write(html_endpart) # write final closed tags;
report.close()
return parser.parse_args().image, parser.parse_args(
).output, parser.parse_args().wordlist
def steghide(password):
from subprocess import call, DEVNULL
cmd = 'steghide extract -sf {0} -fx {1} -p {2}', format(
image, output, password)
if call(cmd.split(), stdout=DEVNULL, stderr=DEVNULL) == 0:
print('[=] Password: {}\n[Ctrl + C] to stop'.format(password))
if __name__ == '__main__':
banner()
image, output, wordlist = parseArgs()
pool = pool()
start = time()
pool.map(
steghide,
[password.rstrip() for password in open(wordlist, errors='ignore')])
totalTime = time() - start
timeFormat = 'seconds'
if (totalTime >= 60):
totalTime = totalTime / 60
timeFormat = 'minutes'
if (totalTime >= 3600):
totalTime = totalTime / 60
timeFormat = 'hours'
print('[=] Finished : {0:.f2} {1}'.format(totalTime, timeFormat))
'Collection': collection,
'PatientID': PatientID,
'StudyInstanceUID': StudyInstanceUID,
'format': 'JSON'
})
return ujson.loads(response.text)
def getMRI(SeriesInstanceUID, filename):
response = rq.get(API_ADDRESS + 'getImage/',
headers={
'X-Mashape-Key': PROD_MASHAPE_API_KEY,
'api_key': TCIA_API_KEY,
'Accept': 'application/json'
},
params={'SeriesInstanceUID': SeriesInstanceUID},
stream=True)
with open(filename, 'wb') as f:
for chunk in response.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
pool = mp.pool(mp.cpu_count)
study = getStudy(STUDY)
print(study)
series = getSeries(study[0]['PatientID'],
study[0]['StudyInstanceUID'],
collection=STUDY)
print(series)
'Accept': 'application/json'}, params={'Collection': collection,
'format': 'JSON'})
return ujson.loads(response.text)
def getSeries(PatientID, StudyInstanceUID, collection=STUDY):
response = rq.get(API_ADDRESS+'getSeries/',
headers={'X-Mashape-Key': PROD_MASHAPE_API_KEY, 'api_key': TCIA_API_KEY,
'Accept': 'application/json'}, params={'Collection': collection,
'PatientID': PatientID,
'StudyInstanceUID': StudyInstanceUID,
'format': 'JSON'})
return ujson.loads(response.text)
def getMRI(SeriesInstanceUID, filename):
response = rq.get(API_ADDRESS+'getImage/',
headers={'X-Mashape-Key': PROD_MASHAPE_API_KEY, 'api_key': TCIA_API_KEY,
'Accept': 'application/json'}, params={
'SeriesInstanceUID': SeriesInstanceUID}, stream=True)
with open(filename, 'wb') as f:
for chunk in response.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
pool = mp.pool(mp.cpu_count)
study = getStudy(STUDY)
print(study)
series = getSeries(study[0]['PatientID'], study[0]['StudyInstanceUID'],
collection=STUDY)
print(series)
import multiprocessing as mp
import time
import random
# one core needs data from other core
# need them to communication
# Message Passing Interface MPI
import os
def really_time_consuming(x):
time.sleep(0.5)
return random.random()
if __name__ == '__main__':
fanal_value = 0
trails = list(range(350))
myPool = mp.pool(4)
for i in trails:
final_value += really_time_consuming(i)
expt_results = myPool.map(really_time_consuming, trails)
final_value = sum(expt_results)
print(os.get_pid())
def main():
photos = get_photos()
with pool(workers=10) as process_pool:
process_pool.map(photos, download_photo)
def networker(routers):
workers = pool(6)
workers.map(mac_ip, map(lambda x: x[0], routers))
workers.close()
process = globals().get(argv[1])
except IndexError:
print "Must supply a process to run."
exit(0)
except AttributeError:
print argv[1], "isn't a process."
exit(0)
if argv[-1] == 'loop':
runs.pop(-1)
print "looping..."
loop(process, runs)
elif argv[-1] == 'pool':
runs.pop(-1)
print "multiprocessing"
pool(process, runs)
elif argv[-1] == 'fast':
processors = 12
pool(process, runs)
elif argv[-1] == 'all':
pool(process, runs)
elif 'r11_' in argv[-1] and len(argv) == 3:
print "straight processing..."
process(argv[-1])
elif 'r11_' in argv[-1]:
print "multiprocessing..."
pool(process, runs)
else:
print "I'm sorry, Aaron. I'm afraid I can't do that."
exit('goodbye')
from multiprocessing import pool
import time
CONT = 500000000
def contdown(n):
while n > 0:
n -= 1
if __name__ == '__main':
pool = pool(processes=2)
start = time.time()
r1 = pool.applay_asnc(contdown, [CONT // 2])
r2 = pool.applay_asnc(contdown, [CONT // 2])
pool.close()
pool.join()
end = time.time()
print('time taken in second-', end - start)
