def run(self):
from production import util
# copy the script to the temp folder and replace the shebang
file_dir = os.path.dirname(os.path.abspath(__file__))
util.copy_and_replace(
os.path.join(file_dir, 'initial_features.py'),
os.path.join(self.tmp_folder, 'initial_features.py'))
with open(self.config_path) as f:
config = json.load(f)
block_shape = config['block_shape']
offsets = config['affinity_offsets']
roi = config.get('roi', None)
# hardcoded keys
graph_key = 'graph'
out_key = 'features'
# create the outpuy files
f_graph = z5py.File(self.graph_path, use_zarr_format=False)
shape = f_graph.attrs['shape']
ds_graph = f_graph[graph_key]
n_edges = ds_graph.attrs['numberOfEdges']
f_out = z5py.File(self.out_path, use_zarr_format=False)
f_out.require_group('blocks')
# chunk size = 64**3
chunk_size = min(262144, n_edges)
f_out.require_dataset(out_key,
dtype='float64',
shape=(n_edges, 10),
chunks=(chunk_size, 1),
compression='gzip')
# get number of blocks
blocking = nifty.tools.blocking([0, 0, 0], shape, block_shape)
# check if we have a roi and adjuse the block list if we do
if roi is None:
n_blocks = blocking.numberOfBlocks
block_list = list(range(n_blocks))
else:
block_list = blocking.getBlockIdsOverlappingBoundingBox(
roi[0], roi[1], [0, 0, 0]).tolist()
n_blocks = len(block_list)
# find the actual number of jobs and prepare job configs
n_jobs = min(n_blocks, self.max_jobs)
self._prepare_jobs(n_jobs, block_list, offsets)
# submit the jobs
if self.run_local:
# this only works in python 3 ?!
with futures.ProcessPoolExecutor(n_jobs) as tp:
tasks = [
tp.submit(self._submit_job, job_id)
for job_id in range(n_jobs)
]
[t.result() for t in tasks]
else:
for job_id in range(n_jobs):
self._submit_job(job_id)
# wait till all jobs are finished
if not self.run_local:
util.wait_for_jobs('papec')
# check the job outputs
processed_jobs, times = self._collect_outputs(n_jobs)
assert len(processed_jobs) == len(times)
success = len(processed_jobs) == n_jobs
# write output file if we succeed, otherwise write partial
# success to different file and raise exception
if success:
out = self.output()
# TODO does 'out' support with job?
fres = out.open('w')
json.dump({'times': times}, fres)
fres.close()
else:
log_path = os.path.join(self.tmp_folder,
'initial_features_partial.json')
with open(log_path, 'w') as out:
json.dump({
'times': times,
'processed_jobs': processed_jobs
}, out)
raise RuntimeError(
"InitialFeatureTask failed, %i / %i jobs processed," %
(len(processed_jobs), n_jobs) +
"serialized partial results to %s" % log_path)
parser = argparse.ArgumentParser()
parser.add_argument('-r',
type=float,
default=0.0,
help='ratio of missing items')
args = parser.parse_args()
RATIO = 1 - args.r
with open('../ml-1m/ratining.csv') as fin:
fin.readline()
ALL_ITEMS = {line.strip().split(',')[1] for line in fin}
with open('item_index2entity_id_ratio_{:.2f}.txt'.format(RATIO), 'wt') as fout:
future_to_movie = {}
with cf.ProcessPoolExecutor(max_workers=50) as executor:
with open('./MappingMovielens2DBpedia-1.2.tsv') as fin:
fin.readline()
for line in fin:
movie_id, _, dbpedia_id = line.strip().split('\t')
if movie_id in ALL_ITEMS:
future_to_movie[executor.submit(get_mapping,
dbpedia_id)] = movie_id
output: list = []
for future in cf.as_completed(future_to_movie):
movie_id = future_to_movie[future]
kg_id = future.result()
if kg_id:
output.append('{}\t{}'.format(movie_id, kg_id))
random.shuffle(output)
fout.write("\n".join(output[:int(len(output) * (RATIO))]))
def estimate_pi(num_steps=20000, step_size=0.1):
_, s = markov_chain(num_steps, step_size)
return np.mean(s)
NUM_STEPS = 20000
# Default step size
chain, samples = markov_chain(NUM_STEPS, 0.1, show_plot=True, name="default")
# Optimal step size
STEP_SIZE = 1.1812
markov_chain(NUM_STEPS, STEP_SIZE, show_plot=True, name="best_step_size")
with futures.ProcessPoolExecutor() as executor:
fs = [
executor.submit(estimate_pi, NUM_STEPS, STEP_SIZE) for _ in range(100)
]
m = np.array([f.result() for f in fs])
print("Empirical variance: {}".format(np.sqrt(np.var(m))))
# batching procedure to estimate the variance
def batching(samples, variance):
if len(samples) <= 1:
return variance
if len(samples) % 2 != 0:
import os
import time
import sys
from concurrent import futures
def to_do(info):
st = time.time()
for i in range(10000000):
pass
print(f"{info[0]} and {info[1]} took {time.time()-st} seconds.")
return info[0]
start_time = time.time()
MAX_WORKERS = 15
param_list = []
for i in range(15):
param_list.append(('text%s' % i, 'info%s' % i))
workers = min(MAX_WORKERS, len(param_list))
# with 默认会等所有任务都完成才返回，所以这里会阻塞
with futures.ProcessPoolExecutor(workers) as executor:
results = executor.map(to_do, sorted(param_list))
# 打印所有
for index, result in enumerate(results, start=1):
print(f"{index} result is: {result}")
print(time.time() - start_time)
# 耗时0.3704512119293213s， 而线程版本需要14.935384511947632s
def run_tests(all_tests, log_name_base, extra_args):
global stop, executor, futures
txtname = log_name_base + '.txt'
xmlname = log_name_base + '.xml'
logfile = open(txtname, 'w', encoding="utf_8")
junit_root = ET.Element('testsuites')
conf_time = 0
build_time = 0
test_time = 0
passing_tests = 0
failing_tests = 0
skipped_tests = 0
try:
# This fails in some CI environments for unknown reasons.
num_workers = multiprocessing.cpu_count()
except Exception as e:
print(
'Could not determine number of CPUs due to the following reason:' +
str(e))
print('Defaulting to using only one process')
num_workers = 1
executor = conc.ProcessPoolExecutor(max_workers=num_workers)
for name, test_cases, skipped in all_tests:
current_suite = ET.SubElement(junit_root, 'testsuite', {
'name': name,
'tests': str(len(test_cases))
})
if skipped:
print('\nNot running %s tests.\n' % name)
else:
print('\nRunning %s tests.\n' % name)
futures = []
for t in test_cases:
# Jenkins screws us over by automatically sorting test cases by name
# and getting it wrong by not doing logical number sorting.
(testnum, testbase) = os.path.split(t)[-1].split(' ', 1)
testname = '%.3d %s' % (int(testnum), testbase)
should_fail = False
if name.startswith('failing'):
should_fail = name.split('failing-')[1]
result = executor.submit(run_test, skipped, t, extra_args,
unity_flags + backend_flags,
compile_commands, should_fail)
futures.append((testname, t, result))
for (testname, t, result) in futures:
result = result.result()
if result is None or 'MESON_SKIP_TEST' in result.stdo:
print('Skipping:', t)
current_test = ET.SubElement(current_suite, 'testcase', {
'name': testname,
'classname': name
})
ET.SubElement(current_test, 'skipped', {})
skipped_tests += 1
else:
without_install = "" if len(
install_commands) > 0 else " (without install)"
if result.msg != '':
print('Failed test%s: %s' % (without_install, t))
print('Reason:', result.msg)
failing_tests += 1
failing_logs.append(result.stdo)
failing_logs.append(result.stde)
else:
print('Succeeded test%s: %s' % (without_install, t))
passing_tests += 1
conf_time += result.conftime
build_time += result.buildtime
test_time += result.testtime
total_time = conf_time + build_time + test_time
log_text_file(logfile, t, result.stdo, result.stde)
current_test = ET.SubElement(
current_suite, 'testcase', {
'name': testname,
'classname': name,
'time': '%.3f' % total_time
})
if result.msg != '':
ET.SubElement(current_test, 'failure',
{'message': result.msg})
stdoel = ET.SubElement(current_test, 'system-out')
stdoel.text = result.stdo
stdeel = ET.SubElement(current_test, 'system-err')
stdeel.text = result.stde
print("\nTotal configuration time: %.2fs" % conf_time)
print("Total build time: %.2fs" % build_time)
print("Total test time: %.2fs" % test_time)
ET.ElementTree(element=junit_root).write(xmlname,
xml_declaration=True,
encoding='UTF-8')
return (passing_tests, failing_tests, skipped_tests)
def calc_csr_kick(
beam,
charges,
Np=None,
gamma=None,
rho=None,
Nz=100,
sigma_z=1e-3,
Nx=100,
sigma_x=1e-3,
reuse_psi_grids=False,
psi_s_grid_old=None,
psi_x_grid_old=None,
verbose=True,
):
"""
"""
(x_b, xp_b, y_b, yp_b, z_b, zp_b) = beam
zx_positions = np.stack((z_b, x_b)).T
# Fix the grid here
# The grid needs to enclose most particles in z-x space
mins = np.array([-6 * sigma_z, -6 * sigma_x]) # Lower bounds of the grid
maxs = np.array([6 * sigma_z, 6 * sigma_x]) # Upper bounds of the grid
sizes = np.array([Nz, Nx])
(zmin, xmin) = mins
(zmax, xmax) = maxs
(Nz, Nx) = sizes
(dz, dx) = (maxs - mins) / (sizes - 1) # grid steps
# indexes, contrib = split_particles(zx_positions, charges, mins, maxs, sizes)
# t1 = time.time();
# charge_grid = deposit_particles(Np, sizes, indexes, contrib)
# t2 = time.time();
# Remi's fast code
t1 = time.time()
charge_grid = histogram_cic_2d(z_b, x_b, charges, Nz, zmin, zmax, Nx, xmin, xmax)
t2 = time.time()
# Normalize the grid so its integral is unity
norm = np.sum(charge_grid) * dz * dx
lambda_grid = charge_grid / norm
# Apply savgol filter
lambda_grid_filtered = np.array(
[savgol_filter(lambda_grid[:, i], 13, 2) for i in np.arange(Nx)]
).T
# Differentiation in z
lambda_grid_filtered_prime = central_difference_z(
lambda_grid_filtered, Nz, Nx, dz, order=1
)
zvec = np.linspace(zmin, zmax, Nz)
xvec = np.linspace(xmin, xmax, Nx)
beta = (1 - 1 / gamma ** 2) ** (1 / 2)
t3 = time.time()
if reuse_psi_grids == True:
psi_s_grid = psi_s_grid_old
psi_x_grid = psi_x_grid_old
else:
# Creating the potential grids
zvec2 = np.linspace(2 * zmin, 2 * zmax, 2 * Nz)
xvec2 = np.linspace(2 * xmin, 2 * xmax, 2 * Nx)
zm2, xm2 = np.meshgrid(zvec2, xvec2, indexing="ij")
# psi_s_grid = psi_s(zm2,xm2,beta)
beta_grid = beta * np.ones(zm2.shape)
with cf.ProcessPoolExecutor(max_workers=12) as executor:
temp = executor.map(psi_s, zm2 / 2 / rho, xm2, beta_grid)
psi_s_grid = np.array(list(temp))
temp2 = executor.map(psi_x, zm2 / 2 / rho, xm2, beta_grid)
psi_x_grid = np.array(list(temp2))
t4 = time.time()
if verbose:
print("Depositting particles takes:", t2 - t1, "s")
print("Computing potential grids take:", t4 - t3, "s")
# Compute the wake via 2d convolution
conv_s = oaconvolve(lambda_grid_filtered_prime, psi_s_grid, mode="same")
conv_x = oaconvolve(lambda_grid_filtered_prime, psi_x_grid, mode="same")
Ws_grid = (beta ** 2 / rho) * (conv_s) * (dz * dx)
Wx_grid = (beta ** 2 / rho) * (conv_x) * (dz * dx)
# Interpolate Ws and Wx everywhere within the grid
Ws_interp = RectBivariateSpline(zvec, xvec, Ws_grid)
Wx_interp = RectBivariateSpline(zvec, xvec, Wx_grid)
r_e = 2.8179403227e-15
q_e = 1.602176634e-19
Nb = np.sum(charge_grid) / q_e
kick_factor = r_e * Nb / gamma
# Calculate the kicks at the paritcle locations
delta_kick = kick_factor * Ws_interp.ev(z_b, x_b)
xp_kick = kick_factor * Wx_interp.ev(z_b, x_b)
return {
"zvec": zvec,
"xvec": xvec,
"delta_kick": delta_kick,
"xp_kick": xp_kick,
"Ws_grid": Ws_grid,
"Wx_grid": Wx_grid,
"psi_s_grid": psi_s_grid,
"psi_x_grid": psi_x_grid,
"charge_grid": charge_grid,
}
def spread(start_locations, frictiondist, friction):
""" Total friction of the shortest accumulated friction path over a map with friction values from a source cell to cell under consideration
:param start_locations: starting locations
:type start_locations: Property
:param frictiondist: initial friction
:type frictiondist: Property
:param friction: friction per cell
:type friction: Property
:returns: a property with total friction values
:rtype: Property
For concepts of this operation calculated on each agent see
https://pcraster.geo.uu.nl/pcraster/4.3.0/documentation/pcraster_manual/sphinx/op_spread.html
"""
result_prop = Property('emptyspreadname', start_locations.pset_uuid,
start_locations.space_domain,
start_locations.shapes)
todo = []
for idx in start_locations.values().values.keys():
start_locations_values = start_locations.values().values[idx]
frictiondist_values = frictiondist.values().values[idx]
friction_values = friction.values().values[idx]
west = start_locations.space_domain.p1.xcoord[idx]
north = start_locations.space_domain.p1.ycoord[idx]
rows = int(start_locations.space_domain.row_discr[idx])
cols = int(start_locations.space_domain.col_discr[idx])
cellsize = (start_locations.space_domain.p2.xcoord[idx] - west) / cols
clone = (rows, cols, cellsize, west, north)
item = (idx, start_locations_values, frictiondist_values,
friction_values, clone)
todo.append(item)
cpus = multiprocessing.cpu_count()
tasks = len(todo)
chunks = tasks // cpus
with futures.ProcessPoolExecutor(max_workers=cpus) as ex:
results = ex.map(_pspread, todo, chunksize=chunks)
for result in results:
result_prop.values().values[result[0]] = result[1]
return result_prop
# sequential
for idx in start_locations.values().values.keys():
values = start_locations.values().values[idx]
_set_current_clone(start_locations, idx)
frictiondistvalues = frictiondist.values().values[idx]
frictionvalues = friction.values().values[idx]
arg1_raster = pcraster.numpy2pcr(pcraster.Nominal, values,
-99999) #numpy.nan)
frictiondist_raster = pcraster.numpy2pcr(pcraster.Scalar,
frictiondistvalues, numpy.nan)
friction_raster = pcraster.numpy2pcr(pcraster.Scalar, frictionvalues,
numpy.nan)
result_raster = pcraster.spread(arg1_raster, frictiondist_raster,
friction_raster)
result_item = pcraster.pcr2numpy(result_raster, numpy.nan)
result_prop.values().values[idx] = result_item
return result_prop
def get_dea_strain(self,
imgs,
output_result_image=False,
check_visual_state=False,
title="",
parallel=True):
"""
Detect strain for the given set of images.
:param imgs: A set of images (1 image per DEA)
:param output_result_image: if True, the function returns a visualisation of the measured strain
:param check_visual_state: Checks if any image has deviated too much from its reference
:param title: A title to print on the result image. Can be a list of titles, one for each result image.
:param parallel: Indicate if multiprocessing should be used to process images in parallel
:return: four object containing the strain, center shift, result images, and visual_state for each of the
n input images. 'strain' is a n-by-4 array with the following columns:
[area strain, radial strain X, radial strain Y, average radial strain (sqrt of area strain)]
'center shift' is a n-by-2 array containing the shift in X and Y direction, in pixels
(reference: top left corner).
'result images' is a n-element list of images with a visualization of the measured strain.
'visual' state is a n-element list of ints indicating 1 if an image is OK, or 0 if an image shows great
deviation from its reference image.
"""
n_img = len(imgs)
# TODO: keep old masks if new masks are rubbish
# TODO: make sure the algorithm is robust to Nones
if self._reference_images is not None and n_img != len(
self._reference_images):
raise Exception(
"Need to specify one image for each reference image! Set elements to None where no image is"
"available for a reference")
# indication if any of the samples must be considered an outlier
outlier = np.array([False] * len(imgs))
# check if image deviation is too large
if check_visual_state:
if self._reference_images is not None:
img_dev = self.get_deviation_from_reference(imgs)
self.logging.debug("Image deviations: {}".format(img_dev))
outlier = np.bitwise_or(
outlier,
np.array(img_dev) > self.image_deviation_threshold)
# initialize lists to store results
ellipses = []
if self._exclude_masks is None:
self._exclude_masks = [None] * n_img
masks = []
# get fit for each DEA
if parallel:
# Use executor in with statement to ensure processes are cleaned up promptly
with futures.ProcessPoolExecutor(max_workers=n_img) as executor:
# Start the load operations and mark each future with its index
fits_results = executor.map(self.dea_fit_ellipse, imgs,
self._exclude_masks)
self.logging.debug(
"Ellipse fitting in parallel. Submitted tasks to worker processes."
)
for res in fits_results:
ellipse, mask = res
ellipses.append(ellipse)
masks.append(mask)
else: # don't use multi-processing
self.logging.debug(
"Sequential ellipse fitting (no multi-processing)")
for i in range(n_img):
ellipse, mask = self.dea_fit_ellipse(imgs[i],
self._exclude_masks[i])
ellipses.append(ellipse)
masks.append(mask)
for i in range(n_img):
# mark as outlier if no ellipse was found
if ellipses[i] is None:
outlier[i] = True
# calculate strain and center shift
ellipses_np = ellipses_to_numpy_array(ellipses) # get as numpy array
xy_radii = ellipse_radius_at_angle(ellipses_np, self.query_angles)
centers = ellipses_np[:, 0:2]
# pseudo because we don't bother multiplying by pi; divide by 4 because ellipse uses diameter not radius
pseudo_areas = np.prod(ellipses_np[:, 2:4], axis=1) / 4
# if there is not yet any reference for strain measurements, set these ellipses as the reference
if self._reference_ellipses is None:
if np.any(ellipses_np == np.nan) or any(outlier):
self.logging.critical(
"Invalid reference! Cannot proceed without a valid reference!"
)
raise Exception("The given reference is not valid")
# if not explicitly calling set_reference, print a warning that a new reference is being set
if self.setting_reference:
self.logging.info("Setting new strain reference.")
else:
self.logging.warning(
"No strain reference has been set yet. The given images will be set as reference."
)
self._reference_images = imgs
self._reference_ellipses = ellipses
self._reference_radii = xy_radii
self._reference_centers = centers
self._reference_pseudo_areas = pseudo_areas
title = "Reference" # to be displayed on the result image
# calculate strain and shift
strain = xy_radii / self._reference_radii
strain_area = pseudo_areas / self._reference_pseudo_areas
strain_avg = np.sqrt(strain_area)
strain_all = np.concatenate(
(np.reshape(strain_area,
(-1, 1)), strain, np.reshape(strain_avg, (-1, 1))),
axis=1)
strain_all = strain_all * 100 - 100 # convert to engineering strain
center_shift = centers - self._reference_centers
# check if strain is too large
if check_visual_state:
strain = strain * 100 - 100 # convert to engineering strain
strain_out = np.bitwise_or(
strain > self.outlier_strain_threshold,
strain < self.outlier_negative_strain_threshold)
strain_out = np.any(strain_out, axis=1)
outlier = np.bitwise_or(outlier, strain_out)
# TODO: make it so this doesn't crash if value are None or nan
shift_out = np.abs(center_shift) > self.center_shift_threshold
shift_out = np.any(shift_out, axis=1)
outlier = np.bitwise_or(outlier, shift_out)
visual_state = list(np.invert(outlier).astype(np.uint8))
else:
visual_state = [None] * n_img
# update exclude masks
for i in range(n_img):
# keep from saving this mask if it belongs to an outlier
if masks[i] is not None and not outlier[i]:
self._exclude_masks[i] = masks[i]
if isinstance(title, str):
title = [
title
] * n_img # make into a list with the same title for each image
elif len(title) == 1 and n_img > 1:
title = title * n_img # if single string in list, duplicate to match number of images
res_imgs = None
if output_result_image:
# create a list of flags indicating the DEA state (as determined visually) or non to disable the indicator
res_imgs = [
visualize_result(imgs[i],
ellipses[i],
tuple(xy_radii[i, :]),
tuple(self.query_angles),
self._reference_ellipses[i],
strain_all[i, -1],
title=title[i]) for i in range(n_img)
]
return strain_all, center_shift, res_imgs, visual_state
def main(wxApp=False):
global PRINT_LOCK, FPS, client, img_counter, start, BLUR_THRESHOLD
PRINT_LOCK = threading.Lock()
BLUR_THRESHOLD = Settings.blur_threshold
FPS = Settings.frames_per_second
CREDNTIALS = Settings.json_path
client = vision.ImageAnnotatorClient.from_service_account_json(CREDNTIALS)
img_counter = 0
start = 0
if not wxApp:
wx_app = wx.App(0)
wx_app.MainLoop()
pic_message = wx.BusyInfo("Taking pictures...")
queue = Queue()
img_dict = dict()
try:
shutil.rmtree(Settings.save_path)
except FileNotFoundError:
pass
finally:
os.makedirs(Settings.save_path)
captureImages_thread = threading.Thread(target=captureImages,
args=(queue, 0,
Settings.seconds_to_run))
captureImages_thread.daemon = True
captureImages_thread.start()
for process in range(Settings.number_of_processes):
thread = threading.Thread(target=threader, args=(queue, img_dict))
thread.daemon = True
thread.start()
queue.join()
captureImages_thread.join()
del pic_message
processing_message = wx.BusyInfo("Processing pictures...")
img_dict = OrderedDict(
sorted(img_dict.items(), reverse=True)[:Settings.max_pics_saved])
with cf.ProcessPoolExecutor(Settings.number_of_processes) as ex:
ex.map(finalize_image, img_dict.items())
if len(img_dict.keys()):
max_score = max(img_dict.keys())
final_image = Image.open(f"{Settings.save_path}/image_{max_score}.jpg")
write_to_image(max_score, final_image, 0)
for score_status, score_range in Settings.score_ranges.items():
if int(max_score * 1000) in score_range:
write_to_image(score_status, final_image, 100)
break
del processing_message
final_image.show()
if not wxApp: del wx_app
# if __name__ == '__main__':
# global PRINT_LOCK, FPS, client, img_counter, start, BLUR_THRESHOLD
# PRINT_LOCK = threading.Lock()
# BLUR_THRESHOLD = Settings.blur_threshold
# FPS = Settings.frames_per_second
# CREDNTIALS = Settings.json_path
# client = vision.ImageAnnotatorClient.from_service_account_json(CREDNTIALS)
# img_counter = 0
# start = 0
# main()
def process_way():
"""使用多进程"""
workers = 10
with futures.ProcessPoolExecutor(workers) as executor:
futs = {executor.submit(blocking_way) for _ in range(10)}
return len([fut.result() for fut in futs])
def setUp(self):
self.pool = futures.ProcessPoolExecutor(4)
self.sc = pysparkling.Context(pool=self.pool,
serializer=cloudpickle.dumps,
deserializer=pickle.loads)
def download_many(cc_list): with futures.ProcessPoolExecutor() as executor: #<-스레드가 아니라 프로세스. 프로세스 수는 기본적으로 os.cpu_count()의 반환값을 사용하므로, 보통은 별도로 설정하지 않아도 된다. res=executor.map(download_one, sorted(cc_list)) return len(list(res))
def geraComando(i, j, k):
cont = i * 25 + j * 5 + k
#comando sem l2
# comand = "./dineroIV " + "-l1-usize " + str(capTot[i]) + "K" +" -l1-uassoc " + str(assoc[j]) + " -l1-ubsize " + str(tamLin[k])+ " -l1-uwalloc a " + "-l1-uwback n " + " -l1-uccc " + "-l1-urepl l " + "-maxtrace 101" + " " +"-trname mcf_f2b "+ "-informat s " + ">" +str(cont)+ "_" + str(capTot[i]) + "_" + str(assoc[j]) + "_" + str(tamLin[k]) + ".txt"
#comando com l2
comand = "./dineroIV " + "-l1-usize " + str(
capTot[i]
) + "K" + " -l1-uassoc " + str(assoc[j]) + " -l1-ubsize " + str(
tamLin[k]
) + " -l1-uwalloc a " + "-l1-uwback n " + " -l1-uccc " + "-l1-urepl l " + "-l2-usize " + "256K" + " -l2-uassoc " + str(
assoc[j]
) + " -l2-ubsize " + str(
tamLin[k]
) + " -l2-uwalloc a " + "-l2-uwback n " + " -l2-uccc " + "-l2-urepl l " + "-maxtrace 101" + " " + "-trname mcf_f2b " + "-informat s " + ">" + str(
cont) + "_" + str(capTot[i]) + "_" + str(assoc[j]) + "_" + str(
tamLin[k]) + ".txt"
print(comand)
os.system(comand)
executor = futures.ProcessPoolExecutor()
for i in range(len(capTot)):
for j in range(len(assoc)):
for k in range(len(tamLin)):
executor.submit(geraComando, i, j, k)
#./dineroIV -l1-dsize 2K -l1-isize 2K -l1-ibsize 16 -l1-dbsize 8 -l2-usize 1m -l2-ubsize 64 -l2-usbsize 16 -l2-uassoc 4 -l2-urepl l -l2-ufetch s -l2-uwalloc a -l2-uwback a -informat s -maxtrace 30 -trname ../dineroData/mcf_f2b
#./dineroIV -l1-usize "$i"K -l1-ubsize $k -l1-uassoc $j -l1-uwalloc a -l1-uwback a -l1-uccc -l2-usize 256K -l2-ubsize $k -l2-usbsize 8 -l2-uassoc 4 - -l2-ufetch s -l2-uwalloc a -l2-uwback a -l2-uccc -maxtrace 101 -trname gcc_f2b
theta[i] = fit_spot(spot)
return theta
def fit_spots_parallel(spots, async=False):
n_workers = max(1, int(0.75 * _multiprocessing.cpu_count()))
n_spots = len(spots)
n_tasks = 100 * n_workers
spots_per_task = [
int(n_spots / n_tasks + 1) if _ < n_spots % n_tasks else int(n_spots /
n_tasks)
for _ in range(n_tasks)
]
start_indices = _np.cumsum([0] + spots_per_task[:-1])
fs = []
executor = _futures.ProcessPoolExecutor(n_workers)
for i, n_spots_task in zip(start_indices, spots_per_task):
fs.append(executor.submit(fit_spots, spots[i:i + n_spots_task]))
if async:
return fs
with _tqdm(total=n_tasks, unit='task') as progress_bar:
for f in _futures.as_completed(fs):
progress_bar.update()
return fits_from_futures(fs)
def fit_spots_gpufit(spots):
size = spots.shape[1]
initial_parameters = initial_parameters_gpufit(spots, size)
spots.shape = (len(spots), (size * size))
model_id = gf.ModelID.GAUSS_2D_ELLIPTIC
from concurrent import futures
import os
def task(n):
return (n, os.getpid())
ex = futures.ProcessPoolExecutor(max_workers=2)
results = ex.map(task, range(5, 0, -1))
for n, pid in results:
print('ran task {} in process {}'.format(n, pid))
record = eval(record[1])
print('{} {} {} {} {} {} {} {}'.format(e, record[0], record[1], record[2], record[3], record[4], record[5], record[6]))
JFm.append(record[0])
Jm.append(record[1])
Jr.append(record[2])
Jd.append(record[3])
Fm.append(record[4])
Fr.append(record[5])
Fd.append(record[6])
print('=========> sort with J&F: <===========')
argidx = np.argmax(np.array(JFm))
print('{} {} {} {} {} {} {} {}'.format(epochs[argidx], JFm[argidx], Jm[argidx], Jr[argidx], Jd[argidx], Fm[argidx], Fr[argidx], Fd[argidx]))
print('=========> sort with Jm: <===========')
argidx = np.argmax(np.array(Jm))
print('{} {} {} {} {} {} {} {}'.format(epochs[argidx], JFm[argidx], Jm[argidx], Jr[argidx], Jd[argidx], Fm[argidx], Fr[argidx], Fd[argidx]))
base_path = join('result', args.dataset)
epochs = listdir(base_path)
print('total {} epochs'.format(len(epochs)))
# multi-process evaluation
if args.dataset in ['DAVIS2016', 'DAVIS2017']:
with futures.ProcessPoolExecutor(max_workers=args.num_threads) as executor:
fs = [executor.submit(eval_davis, e) for e in epochs]
print('done')
extract_davis(epochs)
else:
raise ValueError('not supported data')
class LocalJob(BaseJob):
"""Local QISKit SDK Job class.
Attributes:
_executor (futures.Executor): executor to handle asynchronous jobs
"""
if sys.platform == 'darwin':
_executor = futures.ThreadPoolExecutor()
else:
_executor = futures.ProcessPoolExecutor()
def __init__(self, fn, q_job):
super().__init__()
self._q_job = q_job
self._backend_name = q_job.backend.name
self._future = self._executor.submit(fn, q_job)
def result(self, timeout=None):
# pylint: disable=arguments-differ
"""
Get job result. The behavior is the same as the underlying
concurrent Future objects,
https://docs.python.org/3/library/concurrent.futures.html#future-objects
Args:
timeout (float): number of seconds to wait for results.
Returns:
Result: Result object
Raises:
concurrent.futures.TimeoutError: if timeout occured.
concurrent.futures.CancelledError: if job cancelled before completed.
"""
return self._future.result(timeout=timeout)
def cancel(self):
return self._future.cancel()
@property
def status(self):
_status_msg = None
# order is important here
if self.running:
_status = JobStatus.RUNNING
elif not self.done:
_status = JobStatus.QUEUED
elif self.cancelled:
_status = JobStatus.CANCELLED
elif self.done:
_status = JobStatus.DONE
elif self.exception:
_status = JobStatus.ERROR
_status_msg = str(self.exception)
else:
raise LocalJobError('Unexpected behavior of {0}'.format(
self.__class__.__name__))
return {'status': _status, 'status_msg': _status_msg}
@property
def running(self):
return self._future.running()
@property
def done(self):
"""
Returns True if job successfully finished running.
Note behavior is slightly different than Future objects which would
also return true if successfully cancelled.
"""
return self._future.done() and not self._future.cancelled()
@property
def cancelled(self):
return self._future.cancelled()
@property
def exception(self):
"""
Return Exception object if exception occured else None.
Returns:
Exception: exception raised by attempting to run job.
"""
return self._future.exception(timeout=0)
@property
def backend_name(self):
"""
Return backend name used for this job
"""
return self._backend_name
def from_PTB_basic_FS(
cls,
filesys: "fs.base.FS",
name: typing.Optional[str] = None,
glob_str: str = "**/*.psd",
disambiguate_IDs_by_path: bool = False,
version: Version = Version("0.0.0"),
container_version: Version = Version("0.0.0"),
process_num: int = typing.cast(int, abctk.config.CONF_DEFAULT["max_process_num"]),
tqdm_buffer: typing.Optional[typing.TextIO] = None,
uniformly_with_ID: bool = True,
):
# ------------
# 1. Collect files and their info
# ------------
glob_matches = filesys.glob(glob_str)
file_paths = tuple(
match.path for match in glob_matches
)
file_sizes = tuple(
filesys.getinfo(path, ("details", )).size
for path in file_paths
)
file_num = len(file_paths)
file_size_sum = sum(file_sizes)
logger.info(
f"# of files to be loaded: {file_num}, "
f"the total size: {humanize.naturalsize(file_size_sum)}"
)
# ------------
# 2. Making a file reading iterator
# ------------
def _read_file(filesys: "fs.base.FS", path: str):
with filesys.open(path, "r") as f:
return f.read()
# === END ===
file_contents = (
_read_file(filesys, path)
for path in file_paths
) # As iterator, actual IO made delayed
# ------------
# 3. Launch a multiprocessing context
# -----------
result_tree_iters = list()
with cf.ProcessPoolExecutor(
max_workers = process_num
) as executor:
logger.info(f"Multiprocessing pool created, number of processes: {process_num}")
with tqdm.tqdm(
total = file_size_sum,
desc = "Reading & parsing treebank files",
unit = "B",
unit_scale = True,
file = tqdm_buffer,
disable = tqdm_buffer is None,
) as bar:
for _, size, _, trees in executor.map(
functools.partial(
_load_file,
disambiguate_IDs_by_path = disambiguate_IDs_by_path,
uniformly_with_ID = uniformly_with_ID,
),
zip(
file_paths,
file_sizes,
file_contents,
)
):
bar.update(size)
result_tree_iters.append(trees)
return cls(
name if name is not None else str(filesys),
version,
container_version,
index = dict(itertools.chain.from_iterable(result_tree_iters))
)
return matrix
def filter_cluser(DS):
DS = DS_path + DS
index_path = DS + '/giggle_index'
results = ''
ith_file = 0
files = []
for file in sorted(os.listdir(DS)):
if not file.endswith('.gz'):
continue
ith_file += 1
files += [file]
file_path = os.path.join(DS, file)
cmdline = f'giggle search -i {index_path} -q {file_path} -s'
result = subprocess.check_output(cmdline,
shell=True).decode('utf-8').replace(
'\t\n', '\n')
results = results + file + '\t' + (f'\n{file}\t').join(
result.split("\n")[1:-1]) + "\n"
matrix = pd.DataFrame([], index=files)
if ith_file >= 3:
matrix = inconsistent_filter(results, files)
return matrix, DS.replace(DS_path, '')
with futures.ProcessPoolExecutor(max_workers=4) as executor:
for future in executor.map(filter_cluser):
future.to_csv(args.output, index=False)
im = im.resize((100, 100))
except Exception as e:
print(e)
is_invalid = True
if is_invalid:
shutil.move(file_path, dst_out_path)
if __name__=='__main__':
cid3_paths = os.listdir(file_src)
path_num = len(cid3_paths)
thread_num = 8
with futures.ProcessPoolExecutor(max_workers=thread_num) as executor:
future_list = list()
for i in range(thread_num):
begin = int(path_num / thread_num * i)
end = int(path_num / thread_num * (i + 1))
thread_paths=cid3_paths[begin:end]
future_list.append(executor.submit(split_similarpics, thread_paths))
# future_list.append(executor.submit(split_invalid_pic, thread_paths))
# future_list.append(executor.submit(split_over_num, thread_paths))
for future in futures.as_completed(future_list):
if future.exception():
print(future.exception())
def execute(self):
"""
Raises:
NotADirectoryError: When the event data directory does not exist.
FileNotFoundError: When the the shake_result HDF file does not
exist.
"""
install_path, data_path = get_config_paths()
datadir = os.path.join(data_path, self._eventid, 'current', 'products')
if not os.path.isdir(datadir):
raise NotADirectoryError('%s is not a valid directory.' % datadir)
datafile = os.path.join(datadir, 'shake_result.hdf')
if not os.path.isfile(datafile):
raise FileNotFoundError('%s does not exist.' % datafile)
# Open the ShakeMapOutputContainer and extract the data
container = ShakeMapOutputContainer.load(datafile)
if container.getDataType() != 'grid':
raise NotImplementedError('mapping module can only operate on '
'gridded data, not sets of points')
# get the path to the products.conf file, load the config
config_file = os.path.join(install_path, 'config', 'products.conf')
spec_file = get_configspec('products')
validator = get_custom_validator()
config = ConfigObj(config_file, configspec=spec_file)
results = config.validate(validator)
check_extra_values(config, self.logger)
if not isinstance(results, bool) or not results:
config_error(config, results)
# create contour files
self.logger.debug('Mapping...')
# get the filter size from the products.conf
filter_size = config['products']['contour']['filter_size']
# get the operator setting from config
operator = config['products']['mapping']['operator']
# get all of the pieces needed for the mapping functions
layers = config['products']['mapping']['layers']
if 'topography' in layers and layers['topography'] != '':
topofile = layers['topography']
else:
topofile = None
if 'roads' in layers and layers['roads'] != '':
roadfile = layers['roads']
else:
roadfile = None
if 'faults' in layers and layers['faults'] != '':
faultfile = layers['faults']
else:
faultfile = None
if 'countries' in layers and layers['countries'] != '':
countries_file = layers['countries']
else:
countries_file = None
if 'states_provs' in layers and layers['states_provs'] != '':
states_provs_file = layers['states_provs']
else:
states_provs_file = None
if 'oceans' in layers and layers['oceans'] != '':
oceans_file = layers['oceans']
else:
oceans_file = None
if 'lakes' in layers and layers['lakes'] != '':
lakes_file = layers['lakes']
else:
lakes_file = None
# Get the number of parallel workers
max_workers = config['products']['mapping']['max_workers']
# Reading HDF5 files currently takes a long time, due to poor
# programming in MapIO. To save us some time until that issue is
# resolved, we'll coarsely subset the topo grid once here and pass
# it into both mapping functions
# get the bounds of the map
info = container.getMetadata()
xmin = info['output']['map_information']['min']['longitude']
xmax = info['output']['map_information']['max']['longitude']
ymin = info['output']['map_information']['min']['latitude']
ymax = info['output']['map_information']['max']['latitude']
dy = float(
info['output']['map_information']['grid_spacing']['latitude'])
dx = float(
info['output']['map_information']['grid_spacing']['longitude'])
padx = 5 * dx
pady = 5 * dy
sxmin = float(xmin) - padx
sxmax = float(xmax) + padx
symin = float(ymin) - pady
symax = float(ymax) + pady
sampledict = GeoDict.createDictFromBox(sxmin, sxmax, symin, symax, dx,
dy)
if topofile:
topogrid = read(topofile,
samplegeodict=sampledict,
resample=False,
doPadding=True,
padValue=0.0)
else:
tdata = np.full([sampledict.ny, sampledict.nx], 0.0)
topogrid = Grid2D(data=tdata, geodict=sampledict)
model_config = container.getConfig()
imtlist = container.getIMTs()
textfile = os.path.join(
get_data_path(), 'mapping',
'map_strings.' + config['products']['mapping']['language'])
text_dict = get_text_strings(textfile)
if config['products']['mapping']['fontfamily'] != '':
matplotlib.rcParams['font.family'] = \
config['products']['mapping']['fontfamily']
matplotlib.rcParams['axes.unicode_minus'] = False
allcities = Cities.fromDefault()
states_provs = None
countries = None
oceans = None
lakes = None
faults = None
roads = None
if states_provs_file is not None:
states_provs = ShapelyFeature(
Reader(states_provs_file).geometries(),
ccrs.PlateCarree(),
facecolor='none')
elif 'CALLED_FROM_PYTEST' not in os.environ:
states_provs = cfeature.NaturalEarthFeature(
category='cultural',
name='admin_1_states_provinces_lines',
scale='10m',
facecolor='none')
# The feature constructor doesn't necessarily download the
# data, but we want it to so that multiple threads don't
# try to do it at once when they actually access the data.
# So below we just call the geometries() method to trigger
# the download if necessary.
_ = states_provs.geometries()
if countries_file is not None:
countries = ShapelyFeature(Reader(countries_file).geometries(),
ccrs.PlateCarree(),
facecolor='none')
elif 'CALLED_FROM_PYTEST' not in os.environ:
countries = cfeature.NaturalEarthFeature(category='cultural',
name='admin_0_countries',
scale='10m',
facecolor='none')
_ = countries.geometries()
if oceans_file is not None:
oceans = ShapelyFeature(Reader(oceans_file).geometries(),
ccrs.PlateCarree(),
facecolor=WATERCOLOR)
elif 'CALLED_FROM_PYTEST' not in os.environ:
oceans = cfeature.NaturalEarthFeature(category='physical',
name='ocean',
scale='10m',
facecolor=WATERCOLOR)
_ = oceans.geometries()
if lakes_file is not None:
lakes = ShapelyFeature(Reader(lakes_file).geometries(),
ccrs.PlateCarree(),
facecolor=WATERCOLOR)
elif 'CALLED_FROM_PYTEST' not in os.environ:
lakes = cfeature.NaturalEarthFeature(category='physical',
name='lakes',
scale='10m',
facecolor=WATERCOLOR)
_ = lakes.geometries()
if faultfile is not None:
faults = ShapelyFeature(Reader(faultfile).geometries(),
ccrs.PlateCarree(),
facecolor='none')
if roadfile is not None:
roads = ShapelyFeature(Reader(roadfile).geometries(),
ccrs.PlateCarree(),
facecolor='none')
alist = []
llogo = config['products']['mapping'].get('license_logo') or None
ltext = config['products']['mapping'].get('license_text') or None
for imtype in imtlist:
component, imtype = imtype.split('/')
comp = container.getComponents(imtype)[0]
d = {
'imtype': imtype,
'topogrid': topogrid,
'allcities': allcities,
'states_provinces': states_provs,
'countries': countries,
'oceans': oceans,
'lakes': lakes,
'roads': roads,
'faults': faults,
'datadir': datadir,
'operator': operator,
'filter_size': filter_size,
'info': info,
'component': comp,
'imtdict': container.getIMTGrids(imtype, comp),
'ruptdict': copy.deepcopy(container.getRuptureDict()),
'stationdict': container.getStationDict(),
'config': model_config,
'tdict': text_dict,
'display_magnitude': self.display_magnitude,
'pdf_dpi': config['products']['mapping']['pdf_dpi'],
'img_dpi': config['products']['mapping']['img_dpi'],
'license_logo': llogo,
'license_text': ltext,
}
alist.append(d)
if imtype == 'MMI':
g = copy.deepcopy(d)
g['imtype'] = 'thumbnail'
alist.append(g)
h = copy.deepcopy(d)
h['imtype'] = 'overlay'
alist.append(h)
self.contents.addFile('intensityMap', 'Intensity Map',
'Map of macroseismic intensity.',
'intensity.jpg', 'image/jpeg')
self.contents.addFile('intensityMap', 'Intensity Map',
'Map of macroseismic intensity.',
'intensity.pdf', 'application/pdf')
self.contents.addFile('intensityThumbnail',
'Intensity Thumbnail',
'Thumbnail of intensity map.',
'pin-thumbnail.png', 'image/png')
self.contents.addFile(
'intensityOverlay', 'Intensity Overlay and World File',
'Macroseismic intensity rendered as a '
'PNG overlay and associated world file',
'intensity_overlay.png', 'image/png')
self.contents.addFile(
'intensityOverlay', 'Intensity Overlay and World File',
'Macroseismic intensity rendered as a '
'PNG overlay and associated world file',
'intensity_overlay.pngw', 'text/plain')
else:
fileimt = oq_to_file(imtype)
self.contents.addFile(fileimt + 'Map',
fileimt.upper() + ' Map',
'Map of ' + imtype + '.',
fileimt + '.jpg', 'image/jpeg')
self.contents.addFile(fileimt + 'Map',
fileimt.upper() + ' Map',
'Map of ' + imtype + '.',
fileimt + '.pdf', 'application/pdf')
if max_workers > 0:
with cf.ProcessPoolExecutor(max_workers=max_workers) as ex:
results = ex.map(make_map, alist)
list(results)
else:
for adict in alist:
make_map(adict)
container.close()
def leaderboard(pool, n, focus='hydra.py'):
''
# Saving data from games
data = {name: defaultdict(int) for name in pool}
# Multiprocessing games
with processor.ProcessPoolExecutor() as executor:
# Saving data
outputs = []
games = []
# Generating and scheduling games
for player1 in pool:
for player2 in pool:
if player1 != player2:
for _ in range(n):
games.append(
executor.submit(new_game, player1, player2))
print(f'{len(games)} games scheduled')
# Executing games in parallel
for game in processor.as_completed(games):
output = game.result()
outputs.append(output)
print(
f"{len(outputs)} games completed: {output['players'][0]} vs {output['players'][1]}"
)
# Updating wins/losses
for output in outputs:
for player in output['players']:
if player == output['winner']:
data[player]['wins'] += 1
elif player == output['loser']:
data[player]['losses'] += 1
# Printing which agents beat focus
if player == focus:
print(f'{focus} was beat by {output["winner"]}')
elif output['draw']:
data[player]['draws'] += 1
data[player]['games'] += 1
data[player]['win%'] = round(
data[player]['wins'] / data[player]['games'], 3)
# Scoring agents (win% or wins - losses)
for player in pool:
data[player]['score'] = data[player]['wins'] - data[player]['losses']
data[player]['score'] = round(data[player]['score'], 3)
# Writing data to CSV file
pool.sort(key=lambda name: data[name]['score'], reverse=True)
with open('leaderboard/leaderboard.csv', 'w') as file:
file.write(f"{','.join(data[pool[0]].keys())}")
for name in pool:
output = f'\n'
for item in data[name]:
output += f'{data[name][item]},'
output = output[:-1]
file.write(output)
# Generating PrettyTable and dumping to file
with open('leaderboard/leaderboard_table.txt', 'w') as file:
table = from_csv(open('leaderboard/leaderboard.csv'))
table.align = 'l'
string = table.get_string().replace('-', '_').replace('+', '|')
file.write(string)
def run(self):
from production import util
# copy the script to the temp folder and replace the shebang
file_dir = os.path.dirname(os.path.abspath(__file__))
util.copy_and_replace(
os.path.join(file_dir, 'solve_subproblems.py'),
os.path.join(self.tmp_folder, 'solve_subproblems.py'))
with open(self.config_path) as f:
config = json.load(f)
initial_block_shape = config['block_shape']
n_threads = config['n_threads']
roi = config.get('roi', None)
# get number of blocks
factor = 2**self.scale
block_shape = [factor * bs for bs in initial_block_shape]
shape = z5py.File(self.graph_path).attrs['shape']
blocking = nifty.tools.blocking([0, 0, 0], shape, block_shape)
# check if we have a roi and adjuse the block list if we do
if roi is None:
n_blocks = blocking.numberOfBlocks
block_list = list(range(n_blocks))
else:
block_list = blocking.getBlockIdsOverlappingBoundingBox(
roi[0], roi[1], [0, 0, 0]).tolist()
n_blocks = len(block_list)
# find the actual number of jobs and prepare job configs
n_jobs = min(n_blocks, self.max_jobs)
self._prepare_jobs(n_jobs, block_list, initial_block_shape, n_threads)
# submit the jobs
if self.run_local:
# this only works in python 3 ?!
with futures.ProcessPoolExecutor(n_jobs) as tp:
tasks = [
tp.submit(self._submit_job, job_id, n_threads)
for job_id in range(n_jobs)
]
[t.result() for t in tasks]
else:
for job_id in range(n_jobs):
self._submit_job(job_id, n_threads)
# wait till all jobs are finished
if not self.run_local:
util.wait_for_jobs('papec')
# check the job outputs
processed_blocks, times = self._collect_outputs(block_list)
assert len(processed_blocks) == len(times)
success = len(processed_blocks) == n_blocks
# write output file if we succeed, otherwise write partial
# success to different file and raise exception
if success:
out = self.output()
# TODO does 'out' support with job?
fres = out.open('w')
json.dump({'times': times}, fres)
fres.close()
else:
log_path = os.path.join(
self.tmp_folder,
'solve_subproblems_s%i_partial.json' % self.scale)
with open(log_path, 'w') as out:
json.dump(
{
'times': times,
'processed_blocks': processed_blocks
}, out)
raise RuntimeError("SolveSubproblemTask failed, "
"%i / %i blocks processed, "
"serialized partial results to %s" %
(len(processed_blocks), n_blocks, log_path))
def download_many(cc_list):
with futures.ProcessPoolExecutor() as executor: # <1>
res = executor.map(download_one, sorted(cc_list))
return len(list(res))
def main():
import os
import sys
model_ext = '.ph'
n_best = 15
ps.Band.load_settings()
is_set_model = False
parser = get_parser()
args, unknownargs = parser.parse_known_args()
path = os.getcwd()
if args.path:
path = os.path.expanduser(args.path)
# print(f'-p: {path}')
# Set model names
names = []
if args.input:
for arg in args.input:
names.extend(arg2names(arg=arg, path=path, ext=model_ext))
is_set_model = True
names = list(set(names)) # with unique values
if len(names) == 0 and not is_set_model: # run for all files in the path
names = ps.path.get_model_names(path, model_ext)
if len(names) == 0:
print(f'PATH: {path}')
print(f'-i: {args.input}')
print('I do not know the models for fitting. Please use the key -i MODEL or -i *R500* ')
parser.print_help()
sys.exit(2)
# Set band names
bnames = []
if args.bnames:
for bname in args.bnames.split('-'):
if not ps.band.is_exist(bname):
print('No such band: ' + bname)
parser.print_help()
sys.exit(2)
bnames.append(bname)
# Get observations
observations = ps.cb.observations(args)
if observations is None:
print('No obs data. Use key: -c: ')
parser.print_help()
sys.exit(2)
# curves
curves_o = merge_obs(observations, ps.SetLightCurve)
vels_o = merge_obs(observations, ps.vel.SetVelocityCurve)
# set bands as observed
if curves_o is not None and len(bnames) == 0:
bnames = [bn for bn in curves_o.BandNames if ps.band.is_exist(bn)]
# Set distance and redshift
is_sigma = args.is_fit_sigmas
z = args.redshift
t_diff = args.t_diff
if args.distance is not None:
print("Fit magnitudes on z={0:F} at distance={1:E}".format(z, args.distance))
if z > 0:
print(" Cosmology D(z)={0:E} Mpc".format(ps.cosmology_D_by_z(z)))
else:
distance = 10 # pc
if z > 0:
distance = ps.cosmology_D_by_z(z) * 1e6
print("Fit magnitudes on z={0:F} with cosmology D(z)={1:E} pc".format(z, distance))
args.distance = distance
if is_sigma:
print("Fit magnitudes with model uncertainties.")
print("Color excess E(B-V) ={:f}".format(args.color_excess))
# Time limits for models
tlim = (0, float('inf'))
if args.tlim:
tlim = list(map(float, args.tlim.replace('\\', '').split(':')))
print('Time limits for models: {}'.format(':'.join(map(str, tlim))))
# The fit engine
fitter = engines(args.engine)
fitter.is_info = args.is_not_quiet # fitter = FitMPFit(is_debug=args.is_not_quiet)
# fitter.is_debug = args.is_not_quiet
if args.is_not_quiet:
fitter.print_parameters()
# The filter results by tshift
if args.dtshift:
dtshift = ps.str2interval(args.dtshift, llim=float("-inf"), rlim=float('inf'))
else:
dtshift = (float("-inf"), float("inf"))
# tshift = 0.
res_models = {}
vels_m = {}
res_chi = {}
if len(names) == 1:
name = names[0]
if args.is_not_quiet:
if tlim is not None:
print("Fitting for model %s %s for %s moments" % (path, name, tlim))
else:
print("Fitting for model %s %s, tweight= %f" % (path, name, args.tweight))
# curves_m = lcf.curves_compute(name, path, bnames, z=args.redshift, distance=args.distance,
# t_beg=tlim[0], t_end=tlim[1], t_diff=t_diff)
# res = fitter.fit_curves(curves_o, curves_m)
if vels_o is None:
curves_m, res, res_full = fit_mfl(args, curves_o, bnames, fitter, name, path, t_diff, tlim, is_sigma)
else:
curves_m, res, vel_m = fit_mfl_vel(args, curves_o, vels_o, bnames, fitter, name, path,
t_diff, tlim, is_sigma, A=args.tweight)
vels_m[name] = vel_m
print("{}: t ime shift = {:.2f}+/-{:.4f} Measure: {:.4f} {}".
format(name, res.tshift, res.tsigma, res.measure, res.comm))
# best_tshift = res.tshift
res_models[name] = curves_m
res_chi[name] = res
res_sorted = res_chi
elif len(names) > 1:
if args.nodes > 1:
print("Run parallel fitting: nodes={}, models {}".format(args.nodes, len(names)))
with futures.ProcessPoolExecutor(max_workers=args.nodes) as executor:
if vels_o is None:
future_to_name = {
executor.submit(fit_mfl, args, curves_o, bnames, fitter, n, path, t_diff, tlim,
is_sigma, is_spline=args.is_spline):
n for n in names
}
else:
future_to_name = {
executor.submit(fit_mfl_vel, args, curves_o, vels_o, bnames, fitter,
n, path, t_diff, tlim, is_sigma, is_spline=args.is_spline):
n for n in names
}
i = 0
for future in futures.as_completed(future_to_name):
i += 1
name = future_to_name[future]
try:
data = future.result()
except Exception as exc:
print('%r generated an exception: %s' % (name, exc))
else:
res_models[name], res, vels_m[name] = data
res_chi[name] = res
print("[{}/{}] {:30s} -> {}".format(i, len(names), name, res.comm))
else:
for i, name in enumerate(names, start=1):
txt = "Fitting [{}] for model {:30s} [{}/{}]".format(fitter.Name, name, i, len(names))
if args.is_not_quiet:
print(txt)
else:
sys.stdout.write(u"\u001b[1000D" + txt)
sys.stdout.flush()
if vels_o is None:
curves_m, res, res_full = fit_mfl(args, curves_o, bnames, fitter, name, path, t_diff, tlim,
is_sigma, is_spline=args.is_spline)
else:
curves_m, res, vel_m = fit_mfl_vel(args, curves_o, vels_o, bnames, fitter, name, path,
t_diff, tlim, is_sigma,
is_spline=args.is_spline, A=args.tweight)
vels_m[name] = vel_m
res_models[name] = curves_m
res_chi[name] = res
# select with dtshift
res_chi_sel = {}
for k, v in res_chi.items():
if dtshift[0] < v.tshift < dtshift[1]:
res_chi_sel[k] = v
res_chi = res_chi_sel
# sort with measure
res_sorted = OrderedDict(sorted(res_chi.items(), key=lambda kv: kv[1].measure))
else:
print("No any data about models. Path: {}".format(path))
parser.print_help()
sys.exit(2)
# print results
print("\n Results (tshift in range:{:.2f} -- {:.2f}".format(dtshift[0], dtshift[1]))
print("N || {:40s} ||{:18s}|| {:10}".format('Model', 'dt+-t_err', 'Measure'))
for i, (k, v) in enumerate(res_sorted.items(), start=1):
print("{} || {:40s} || {:7.2f}+/-{:7.4f} || {:.4f} || {}".format(i, k, v.tshift, v.tsigma, v.measure, v.comm))
if len(res_sorted) >= n_best:
# plot chi squared
plot_squared_grid(res_sorted, path, is_not_quiet=args.is_not_quiet)
plot_chi_par(res_sorted, path)
# plot only Nbest modeles
# while len(res_sorted) > Nbest:
# res_sorted.popitem()
# plot_squared_3d(None, res_sorted, path, p=('R', 'M', 'E'), is_polar=True)
best_mdl, res = ps.first(res_sorted.items())
# res = first(res_sorted.values())[0]
print("Best fit model:")
print("{}: time shift = {:.2f}+/-{:.4f} Measure: {:.4f}".format(best_mdl, res.tshift, res.tsigma, res.measure))
print("{}: ".format(best_mdl), res.comm)
# shift observational data
# curves_o.set_tshift(best_tshift)
# plot only NbestPlot modeles
while len(res_sorted) > args.plotnbest:
res_sorted.popitem()
if vels_o is not None and vels_o.Length > 0:
# vel_o.tshift = best_tshift
fig = plot_curves_vel(curves_o, vels_o, res_models, res_sorted, vels_m)
else:
fig = plot_curves(curves_o, res_models, res_sorted, xlim=tlim)
if args.save_file is not None:
fsave = args.save_file
if len(os.path.dirname(fsave)) == 0:
fsave = os.path.expanduser("~/{}".format(fsave))
if not fsave.endswith('.pdf'):
fsave += '.pdf'
print("Save plot to {0}".format(fsave))
fig.savefig(fsave, bbox_inches='tight')
else:
from matplotlib import pyplot as plt
# plt.subplots_adjust(left=0.07, right=0.96, top=0.97, bottom=0.06)
plt.ion()
plt.show()
plt.pause(0.0001)
print('')
input("===> Hit <return> to quit")
class SimulatorsJob(BaseJob):
"""SimulatorsJob class.
Attributes:
_executor (futures.Executor): executor to handle asynchronous jobs
"""
if sys.platform in ['darwin', 'win32']:
_executor = futures.ThreadPoolExecutor()
else:
_executor = futures.ProcessPoolExecutor()
def __init__(self, backend, job_id, fn, qobj):
super().__init__(backend, job_id)
self._fn = fn
self._qobj = qobj
self._future = None
def submit(self):
"""Submit the job to the backend for execution.
Raises:
QobjValidationError: if the JSON serialization of the Qobj passed
during construction does not validate against the Qobj schema.
JobError: if trying to re-submit the job.
"""
if self._future is not None:
raise JobError("We have already submitted the job!")
validate_qobj_against_schema(self._qobj)
self._future = self._executor.submit(self._fn, self._job_id,
self._qobj)
@requires_submit
def result(self, timeout=None):
# pylint: disable=arguments-differ
"""Get job result. The behavior is the same as the underlying
concurrent Future objects,
https://docs.python.org/3/library/concurrent.futures.html#future-objects
Args:
timeout (float): number of seconds to wait for results.
Returns:
qiskit.Result: Result object
Raises:
concurrent.futures.TimeoutError: if timeout occurred.
concurrent.futures.CancelledError: if job cancelled before completed.
"""
return self._future.result(timeout=timeout)
@requires_submit
def cancel(self):
return self._future.cancel()
@requires_submit
def status(self):
"""Gets the status of the job by querying the Python's future
Returns:
qiskit.providers.JobStatus: The current JobStatus
Raises:
JobError: If the future is in unexpected state
concurrent.futures.TimeoutError: if timeout occurred.
"""
# The order is important here
if self._future.running():
_status = JobStatus.RUNNING
elif self._future.cancelled():
_status = JobStatus.CANCELLED
elif self._future.done():
_status = JobStatus.DONE if self._future.exception(
) is None else JobStatus.ERROR
else:
# Note: There is an undocumented Future state: PENDING, that seems to show up when
# the job is enqueued, waiting for someone to pick it up. We need to deal with this
# state but there's no public API for it, so we are assuming that if the job is not
# in any of the previous states, is PENDING, ergo INITIALIZING for us.
_status = JobStatus.INITIALIZING
return _status
def backend(self):
"""Return the instance of the backend used for this job."""
return self._backend
def qobj(self):
"""Return the Qobj submitted for this job.
Returns:
Qobj: the Qobj submitted for this job.
"""
return self._qobj
import logging
import time
import concurrent.futures as f
import multiprocessing as m
m.log_to_stderr(logging.DEBUG)
def fn(n):
print("start", n)
time.sleep(1)
return n * n
logging.basicConfig(level=logging.DEBUG)
print("----------------------------------------")
with f.ProcessPoolExecutor() as e:
futs = []
for i in range(7):
futs.append(e.submit(fn, i))
for fut in f.as_completed(futs):
print(fut.result())
print("----------------------------------------")
display_step=1,
save_files=True,
file_name=modelname,
save_model=True,
model_name=modelname,
random_state=1)
# evaluate: niters is the number of times you evaluate the network for
# a single sample. higher niters = better resolved distribution of predictions
niters = 100
bl, bm, bu, results = bmodel.bayesian_predict(x_test,
n_iters=niters,
random_state=1)
# bayesian_predict returns 25, 50, 75 percentile and results dict with all predictions
# evaluate performance on median predictions
acc = sklearn.metrics.accuracy_score(y_test, bm.flatten())
print('Ye bmodel ka Accuracy', acc)
# collect_tree_predictions gathers predictions in results dict
# in a more intuitive way for easy plotting, etc
p = bmodel.collect_tree_predictions(results['predictions'])
a = time()
with cf.ProcessPoolExecutor() as p:
p.submit(model_train1)
p.submit(model_train2)
b = time()
print("Total time taken:", b - a)
def test_max_workers_too_large(self):
with self.assertRaisesRegex(ValueError,
"max_workers must be <= 61"):
futures.ProcessPoolExecutor(max_workers=62)
def update_fitness(INFO):
import concurrent.futures as cf
with cf.ProcessPoolExecutor() as e:
for result_list in e.map(proc_worker, INFO):
for l in result_list:
print(l)
