def main_optical(dates):
saldana = regionstack.regionStack('Saldana', attrs=['S2', 'LC08'])
#saldana.harmonize_L8()
boi = ['red', 'blue', 'green', 'nir', 'swir1', 'swir2']
vars_loc = os.environ['WIN_SVR_DATA'] + 'Saldana/vars/'
for band in boi:
if not os.path.isfile(vars_loc + 'opt_' + band + '.nc'):
## TO-DO add isel for relevant dates
sentinel = saldana.S2[
band] #.isel(time=slice(min(dates)-120?D-security_margin, max(dates)+16D+security_margin))
landsat = saldana.LC08[
band] #.isel(time=slice(min(dates)-120?D, max(dates)+16D))
aligned = xr.align(sentinel, landsat, exclude={'time'})
da = xr.concat([aligned[0], aligned[1]], dim='time')
da.sortby('time').to_netcdf(vars_loc + 'opt_' + band + '.nc')
client = Client(n_workers=12)
client.upload_file('c_Class_Models/interpolatets.py')
files = list(filter(re.compile(r'^opt_.*').search, os.listdir(vars_loc)))
files = list(map(lambda x: vars_loc + x, files))
print('Reading concatenated Dataset')
dataset = xr.open_mfdataset(files,
chunks={
'y': 1000,
'x': 750,
'time': -1
})
print('Concat Dataset\n')
print(dataset)
_location = os.environ['WIN_SVR_DATA'] + 'Saldana/features/'
itp.interpolate_dataset(dataset,
_location,
boi,
date_of_analysis=dates,
der=False)
print('Dataset interpolation done!\n')
def main_radar_text(dates):
saldana = regionstack.regionStack(
'Saldana', attrs=['S1_ASCENDING_GLCM', 'S1_DESCENDING_GLCM'])
boi = [
'VV_ASM', 'VV_Contrast', 'VV_Dissimilarity', 'VV_Energy', 'VV_Entropy',
'VV_GLCMCorrelation', 'VV_GLCMMean', 'VV_GLCMVariance',
'VV_Homogeneity'
]
vars_loc = os.environ['WIN_SVR_DATA'] + 'Saldana/vars/'
for band in boi:
if not os.path.isfile(vars_loc + 'rad_' + band + '.nc'):
asc = saldana.S1_ASCENDING_GLCM[band]
dsc = saldana.S1_DESCENDING_GLCM[band]
da = xr.concat([asc, dsc], dim='time')
print('{} band was concatenated. Writing DataArray'.format(band))
da.to_netcdf(vars_loc + 'rad_' + band + '.nc')
client = Client(n_workers=12)
client.upload_file('b_Temporal_Stack/interpolatets.py')
files = list(filter(re.compile(r'^rad_.*').search, os.listdir(vars_loc)))
files = list(map(lambda x: vars_loc + x, files))
print('Reading concatenated Dataset')
dataset = xr.open_mfdataset(files,
chunks={
'y': 1000,
'x': 750,
'time': -1
})
print('Concat Dataset\n')
print(dataset)
_location = os.environ['WIN_SVR_DATA'] + 'Saldana/features/'
itp.interpolate_dataset(dataset, _location, boi, date_of_analysis=dates)
itp.interpolate_dataset(dataset,
_location,
boi,
date_of_analysis=dates,
der=True)
###############################################################################
###############################################################################
##### Cluster initialization
if parallel:
if cluster == 'distributed':
cluster_ = DISTCLUSTER
else:
nworkers = NLOCALWORKERS
cluster_ = LocalCluster(n_workers=nworkers,
threads_per_worker=1,
memory_limit='5GB')
client = Client(cluster_)
client.upload_file('structure.py')
client.upload_file('decoding_functions.py')
else:
client = None
###############################################################################
for (dataseed, taskvar, monkey, stable, region, subspace, ensemble,
permutes) in product(dataseeds, taskvars, monkeys, stables, regions,
subspaces, ensembles, permutations):
print(
f"monkey:{monkey} | region:{region} | var:{taskvar} | stable:{stable} | subspace:{subspace} | "
f"permutes:{permutes} | ensemble:{ensemble} | seed:{dataseed}")
params_preproc_ens = get_ens_pp(stable)
params_preproc_test = get_test_pp()
parser.add_argument("experiment",
type=str,
help="Name of the experiment being performed")
parser.add_argument("--benchmark",
action="store_true",
help="benchmark results")
args = parser.parse_args()
# Cluster scheduler
cluster = args.scheduler
client = Client(cluster)
print(client)
# Allow workers to use module
client.upload_file("/nfs/paper-big-data-engines/utils.py")
client.upload_file("/nfs/paper-big-data-engines/kmeans/Kmeans.py")
# Read images
paths = crawl_dir(os.path.abspath(args.bb_dir))
paths = db.from_sequence(paths, npartitions=len(paths))
img_rdd = paths.map(lambda p: read_img(p, start=start, args=args))
start_time = time() - start
voxels = img_rdd.map(lambda x: x[1].flatten("F")).flatten()
frequency_pair = voxels.frequencies().compute()
end_time = time() - start
if args.benchmark:
parser.add_argument("experiment",
type=str,
help="Name of the experiment being performed")
parser.add_argument("--benchmark",
action="store_true",
help="benchmark results")
args = parser.parse_args()
# Cluster scheduler
cluster = args.scheduler
client = Client(cluster)
print(client)
# Allow workers to use module
client.upload_file("/nfs/paper-big-data-engines/utils.py")
client.upload_file("/nfs/paper-big-data-engines/histogram/Histogram.py")
from utils import benchmark, crawl_dir, read_img
from Histogram import (
calculate_histogram,
combine_histogram,
flatten,
save_histogram,
)
# Read images
paths = crawl_dir(os.path.abspath(args.bb_dir))
paths = db.from_sequence(paths, npartitions=len(paths))
img = paths.map(lambda p: read_img(p, start=start, args=args))
img = img.map(
parser.add_argument(
"benchmark_dir", help="Directory where thebenchmark files are written."
)
parser.add_argument(
"experiment", type=str, help="Name of the experiment being performed"
)
parser.add_argument("--benchmark", action="store_true", help="benchmark results")
args = parser.parse_args()
# Cluster scheduler
cluster = args.scheduler
client = Client(cluster)
print(client)
client.upload_file("/nfs/paper-big-data-engines/utils.py")
client.upload_file("/nfs/paper-big-data-engines/bidsApp-examples/Example.py")
from Example import run_group, run_participant, site_crawler, subject_crawler
# Retrieve all subject path
subjects = subject_crawler(args.bids_dir)
client.scatter(subjects)
results = list()
for subject in subjects:
results.append(
client.submit(
run_participant,
subject_id=subject[1],
start=start,
args=args,
parser.add_argument("iterations", type=int, help="number of iterations")
parser.add_argument("delay",
type=float,
help="sleep delay during "
"incrementation")
parser.add_argument("--benchmark",
action="store_true",
help="benchmark results")
args = parser.parse_args()
cluster = args.scheduler
client = Client(cluster)
print(client)
client.upload_file(
"/nfs/SOEN-499-Project/utils.py") # Allow workers to use module
client.upload_file("/nfs/SOEN-499-Project/incrementation/Increment.py")
# Read images
paths = crawl_dir(os.path.abspath(args.bb_dir))
paths = db.from_sequence(paths, npartitions=len(paths))
img_rdd = paths.map(lambda p: read_img(p, start=start, args=args))
# Increment the data n time:
for _ in range(args.iterations):
img_rdd = img_rdd.map(
lambda x: increment(x, delay=args.delay, start=start, args=args))
# Save the data
img_rdd = img_rdd.map(lambda x: save_results(x, start=start, args=args))
# +
# Change to parent directory (presumably "Documents")
os.chdir("../../..")
# Path to the CSV dataset files
data_path = 'Documents/Datasets/Outlier_Detection/UniversityOfOregon_SolarAndMeteorologicalData_Eugene/'
project_path = 'Documents/GitHub/eICU-mortality-prediction/'
# -
# Set up local cluster
client = Client('tcp://127.0.0.1:61980')
client
# Upload the utils.py file, so that the Dask cluster has access to relevant auxiliary functions
client.upload_file(f'{project_path}NeuralNetwork.py')
client.upload_file(f'{project_path}utils.py')
client.upload_file(f'{project_path}search_explore.py')
client.upload_file(f'{project_path}data_processing.py')
# **Problem:** Somehow, all works fine if I initialize the Dask client without specifying the tcp address. But if I specify the one obtained from the Jupyter Lab Dask extension, it returns "ModuleNotFoundError: No module named 'torch'"! Perhaps the Jupyter Lab Dask extension is associated to a different Python environment.
#
# **Solution:** Jupyter Lab must be started from within the desired virtual environment's shell.
client.run(os.getcwd)
# ## Loading data
all_files = glob.glob(f'{data_path}/*.txt')
# +
# Mois de modélisation
idmois="_201708"
segmmm=""
# Chemin vers le dossier ou sont les codes du projet
path="/mnt/smb/TAMPON/Igor/RFR_V1/"
path_code = path + "script/"
#path_data = "/mnt/smb/TAMPON/Partages/Moteur_industr/03_PROD/Restitution/EAS/"
path_data = path + "data_src/"
path_rslt = path + "data_rslt/"
os.chdir(path_code)
#import local packages on workers
client.upload_file('K_means.py')
client.upload_file('IntConf.py')
client.upload_file('prediction.py')
client.upload_file('correlation.py')
client.upload_file('data_preparation.py')
client.upload_file('main2.py')
import main2 as workflow
import data_preparation
import K_means as K_means
import IntConf
import prediction
fich_deb = "ech_variables_explicatives" + idmois + ".csv"
#fich_vae = "revenu_client" + segmmm + idmois + ".csv"
return parser.parse_args()
if __name__ == '__main__':
args = parse_args()
# setup the game
game, agents, ratings = game_setup(args.num_agents)
# run the game matches on the cluster
nodes = get_nodes(args.ip_file)
print("Connecting to cluster scheduler {} with workers:".format(nodes[0]))
client = Client(nodes[0] + ':8786')
for worker, cores in client.ncores().items():
print("{:>35} {} cores".format(worker, cores))
client.upload_file('game.py')
start = default_timer()
matches = run_games(game, agents, args.num_matches, client)
check_status(matches)
print("Game run in {:.2f}".format(default_timer() - start))
# here we could do something with failed matches (errors)
# run rating evaluations
start = default_timer()
compute_ratings(matches, ratings)
print("Skills computed in {:.2f}".format(default_timer() - start))
# compute approximate accuracy of ratings
accuracy = estimate_accuracy(agents, ratings)
class Remote(object):
"""
Remote.
Args:
address (str): Remote scheduler address formed by `ip:port`.
tls_ca_file (str, optional): TLS CA certificate file path. Defaults to None.
tls_client_cert (str, optional): TLS certificate file path. Defaults to None.
tls_client_key (str, optional): TLS private key file path. Defaults to None.
require_encryption (bool, optional): Encrypt data exchange. Defaults to False.
Note:
TLS will be enabled only if all three TLS arguments are provided.
Remember to change network protocol to `tls://<address>`.
"""
def __init__(self, address: str,
tls_ca_file: str = None, tls_client_cert: str = None, tls_client_key: str = None,
require_encryption: bool = False):
# authentication
sec = None
if tls_ca_file and tls_client_cert and tls_client_key:
sec = Security(tls_ca_file=tls_ca_file,
tls_client_cert=tls_client_cert,
tls_client_key=tls_client_key,
require_encryption=require_encryption)
# init
self._client = Client(address=address, security=sec)
self._client.register_worker_callbacks(Remote._worker_startup)
@staticmethod
def _worker_startup(dask_worker: Worker):
os.chdir(dask_worker.local_dir)
def add_dependencies(self, files):
"""
Add list of dependencies, order matters.
Args:
files (list): List of dependent files.
"""
# TODO: automatically resolve module dependencies
if isinstance(files, str):
files = [files]
for f in files:
self._client.upload_file(f)
def scatter(self, *args, **kwargs):
"""
Scatter data.
"""
return self._client.scatter(*args, **kwargs)
def submit(self, func, *args, **kwargs):
"""
Submit function and data.
Args:
func (callable): User function.
"""
return self._client.submit(func, *args, **kwargs)
def fetch(self, futures_, **kwargs):
"""
Fetch data of future objects.
Args:
futures_ (list): Future objects.
"""
return self._client.gather(futures_, **kwargs)
def cancel(self, futures_, **kwargs):
"""
Cancel job of future objects.
Args:
futures_ (list): Future objects.
"""
return self._client.cancel(futures_, **kwargs)
def close(self, *args, **kwargs):
"""
Close connection.
"""
return self._client.close(*args, **kwargs)
def main(args):
"""Main function of cellanneal."""
if (args.start_temp is not None
or args.end_temp is not None) and args.auto_temp == 1:
raise Exception(
"when auto_temp is set to 1(default value), starting temperature or ending temperature should not be set manually"
)
if not args.no_parallel:
import dask
from dask.distributed import Client, LocalCluster
if not args.cluster:
cluster = LocalCluster(
n_workers=args.workers,
threads_per_worker=1,
)
client = Client(cluster)
else:
cluster = args.cluster
client = Client(cluster)
client.restart()
cwd = Path(__file__).parent.absolute()
client.upload_file(cwd / 'drawing.py')
client.upload_file(cwd / 'mathhelper.py')
client.upload_file(cwd / 'cell.py')
client.upload_file(cwd / 'colony.py')
client.upload_file(cwd / 'optimization.py')
client.upload_file(cwd / 'drawing.py')
client.upload_file(cwd / 'global_optimization.py')
client.upload_file(cwd / 'main.py')
else:
client = None
lineagefile = None
start = time.time()
try:
config = load_config(args.config)
simulation_config = config["simulation"]
#Maybe better to store the image type in the config file in the first place, instead of using cmd?
if args.graySynthetic == True:
simulation_config["image.type"] = "graySynthetic"
elif args.phaseContrast == True:
simulation_config["image.type"] = "phaseContrastImage"
elif args.binary == True:
simulation_config["image.type"] = "binary"
else:
raise ValueError(
"Invalid Command: Synthetic image type must be specified")
if not args.output.is_dir():
args.output.mkdir()
if not args.bestfit.is_dir():
args.bestfit.mkdir()
if args.residual and not args.residual.is_dir():
args.residual.mkdir()
seed = int(start * 1000) % (2**32)
if args.seed != None:
seed = args.seed
np.random.seed(seed)
print("Seed: {}".format(seed))
celltype = config['global.cellType'].lower()
# setup the colony from a file with the initial properties
lineageframes = LineageFrames()
colony = lineageframes.forward()
imagefiles = get_inputfiles(args)
if args.lineage_file:
load_colony(colony,
args.lineage_file,
config,
initial_frame=imagefiles[0].name)
else:
load_colony(colony, args.initial, config)
cost_diff = (-1, -1)
# open the lineage file for writing
lineagefile = open(args.output / 'lineage.csv', 'w')
header = ['file', 'name']
if celltype == 'bacilli':
header.extend([
'x', 'y', 'width', 'length', 'rotation', "split_alpha",
"opacity"
])
print(','.join(header), file=lineagefile)
if args.debug:
with open(args.debug / 'debug.csv', 'w') as debugfile:
print(','.join([
'window_start', 'window_end', 'pbad_total', 'bad_count',
'temperature', 'total_cost_diff', 'current_iteration',
'total_iterations'
]),
file=debugfile)
if args.global_optimization:
global useDistanceObjective
useDistanceObjective = args.dist
realimages = [
optimization.load_image(imagefile) for imagefile in imagefiles
]
window = config["global_optimizer.window_size"]
if args.lineage_file:
lineage = global_optimization.build_initial_lineage(
imagefiles, args.lineage_file, args.continue_from,
config["simulation"])
else:
lineage = global_optimization.build_initial_lineage(
imagefiles, args.initial, args.continue_from,
config["simulation"])
lineage = global_optimization.find_optimal_simulation_confs(
imagefiles, lineage, realimages, args.continue_from)
sim_start = args.continue_from - args.frame_first
print(sim_start)
shape = realimages[0].shape
synthimages = []
cellmaps = []
distmaps = []
iteration_per_cell = config["iteration_per_cell"]
if not useDistanceObjective:
distmaps = [None] * len(realimages)
for window_start in range(1 - window, len(realimages)):
window_end = window_start + window
print(window_start, window_end)
if window_end <= len(realimages):
# get initial estimate
if window_start >= sim_start:
if window_end > 1:
lineage.copy_forward()
realimage = realimages[window_end - 1]
synthimage, cellmap = optimization.generate_synthetic_image(
lineage.frames[window_end - 1].nodes, shape,
lineage.frames[window_end - 1].simulation_config)
synthimages.append(synthimage)
cellmaps.append(cellmap)
if useDistanceObjective:
distmap = distance_transform_edt(realimage < .5)
distmap /= config[
f'{config["global.cellType"].lower()}.distanceCostDivisor'] * config[
'global.pixelsPerMicron']
distmap += 1
distmaps.append(distmap)
if args.auto_temp == 1 and window_end == 1:
print("auto temperature schedule started")
args.start_temp, args.end_temp = \
global_optimization.auto_temp_schedule(imagefiles, lineage, realimages, synthimages, cellmaps, distmaps, 0, 1, lineagefile, args, config)
print("auto temperature schedule finished")
print("starting temperature is ", args.start_temp,
"ending temperature is ", args.end_temp)
if args.auto_meth == "frame" and optimization.auto_temp_schedule_frame(
window_end, 3):
print("auto temperature schedule restarted")
args.start_temp, args.end_temp = \
global_optimization.auto_temp_schedule(imagefiles, lineage, realimages, synthimages, cellmaps, distmaps, window_start, window_end, lineagefile, args, config)
print("auto temperature schedule finished")
print("starting temperature is ", args.start_temp,
"ending temperature is ", args.end_temp)
if window_start >= sim_start:
if useDistanceObjective:
global_optimization.totalCostDiff = optimization.dist_objective(
realimage, synthimage, distmap, cellmap,
config["overlap.cost"])
else:
global_optimization.totalCostDiff = optimization.objective(
realimage, synthimage, cellmap,
config["overlap.cost"], config["cell.importance"])
lineage, synthimages, distmaps, cellmaps = global_optimization.optimize(
imagefiles,
lineage,
realimages,
synthimages,
cellmaps,
distmaps,
window_start,
window_end,
lineagefile,
args,
config,
iteration_per_cell,
client=client)
if window_start >= 0:
global_optimization.save_lineage(
imagefiles[window_start].name,
lineage.frames[window_start].nodes, lineagefile)
global_optimization.save_output(
imagefiles[window_start].name,
synthimages[window_start], realimages[window_start],
lineage.frames[window_start].nodes, args, config)
return 0
config["simulation"] = optimization.find_optimal_simulation_conf(
config["simulation"], optimization.load_image(imagefiles[0]),
list(colony))
if args.auto_temp == 1:
print("auto temperature schedule started")
args.start_temp, args.end_temp = optimization.auto_temp_schedule(
imagefiles[0], lineageframes.forward(), args, config)
print("auto temperature schedule finished")
print("starting temperature is ", args.start_temp,
"ending temperature is ", args.end_temp)
frame_num = 0
prev_cell_num = len(colony)
for imagefile in imagefiles: # Recomputing temperature when needed
frame_num += 1
if args.auto_meth == "frame":
if optimization.auto_temp_schedule_frame(frame_num, 8):
print("auto temperature schedule started (recomputed)")
args.start_temp, args.end_temp = optimization.auto_temp_schedule(
imagefile, colony, args, config)
print("auto temperature schedule finished")
print("starting temperature is ", args.start_temp,
"ending temperature is ", args.end_temp)
elif args.auto_meth == "factor":
if optimization.auto_temp_schedule_factor(
len(colony), prev_cell_num, 1.1):
print("auto temperature schedule started (recomputed)")
args.start_temp, args.end_temp = optimization.auto_temp_schedule(
imagefile, colony, args, config)
print("auto temperature schedule finished")
print("starting temperature is ", args.start_temp,
"ending temperature is ", args.end_temp)
prev_cell_num = len(colony)
elif args.auto_meth == "const":
if optimization.auto_temp_schedule_const(
len(colony), prev_cell_num, 10):
print("auto temperature schedule started (recomputed)")
args.start_temp, args.end_temp = optimization.auto_temp_schedule(
imagefile, colony, args, config)
print("auto temperature schedule finished")
print("starting temperature is ", args.start_temp,
"ending temperature is ", args.end_temp)
prev_cell_num = len(colony)
elif args.auto_meth == "cost":
print(cost_diff, frame_num,
optimization.auto_temp_shcedule_cost(cost_diff))
if frame_num >= 2 and optimization.auto_temp_shcedule_cost(
cost_diff):
print(
"auto temperature schedule started cost_diff (recomputed)"
)
args.start_temp, args.end_temp = optimization.auto_temp_schedule(
imagefile, colony, args, config)
print("auto temperature schedule finished")
print("starting temperature is ", args.start_temp,
"ending temperature is ", args.end_temp)
colony = optimize(imagefile, lineageframes, args, config, client)
cost_diff = optimization.update_cost_diff(colony, cost_diff)
# flatten modifications and save cell properties
colony.flatten()
for cellnode in colony:
properties = [imagefile.name, cellnode.cell.name]
if celltype == 'bacilli':
properties.extend([
str(cellnode.cell.x),
str(cellnode.cell.y),
str(cellnode.cell.width),
str(cellnode.cell.length),
str(cellnode.cell.rotation)
])
print(','.join(properties), file=lineagefile)
except KeyboardInterrupt as error:
raise error
finally:
if lineagefile:
lineagefile.close()
print(f'{time.time() - start} seconds')
if client and not cluster:
client.shutdown()
return 0
ak.behavior.update(candidate.behavior)
from TTbarResProcessor import TTbarResProcessor
from Filesets import filesets
LoadingUnweightedFiles = False
UsingDaskExecutor = True
if UsingDaskExecutor == True:
if __name__ == "__main__":
tic = time.time()
cluster = LPCCondorCluster()
# minimum > 0: https://github.com/CoffeaTeam/coffea/issues/465
cluster.adapt(minimum=1, maximum=10)
client = Client(cluster)
client.upload_file('TTbarAllHadUproot/TTbarResProcessor.py')
tstart = time.time()
outputs_unweighted = {}
seed = 1234577890
prng = RandomState(seed)
#Chunk = [10000, 100] # [chunksize, maxchunks]
for name,files in filesets.items():
if not LoadingUnweightedFiles:
print('Processing', name)
if not UsingDaskExecutor:
chosen_exec = 'futures'
output = processor.run_uproot_job({name:files},
nbofcluster = int(sys.argv[2])
fromline = 0
toline = 0
csv.register_dialect("tabulation", delimiter='\t')
jobs = []
clusters = []
with open(filename) as f:
nbline = sum(1 for nbline in csv.reader(f, dialect="tabulation"))
f.close()
for cluster in range(nbofcluster):
toline = int((nbline / nbofcluster) * (cluster + 1))
cluster = Client()
clusters.append(cluster)
cluster.set_metadata("clusternb", str(cluster))
cluster.upload_file(filename)
jobs.append(
cluster.submit(network_operator, filename, (fromline, toline)))
jobs.append(
cluster.submit(calc_customer_call, filename, (fromline, toline)))
jobs.append(
cluster.submit(call_proportion, filename, (fromline, toline)))
fromline = int(toline)
call_proportion_data = {}
network_operator_data = {}
calc_customer_call_data = {}
for job in jobs:
result = job.result()
if result[0] == "network_operator":
for key, value in result[1].items():
from pathlib import Path
from dask_cloudprovider_problem import dask_cloudprovider_bug_minimal_example
if __name__ == "__main__":
from dask_cloudprovider import FargateCluster
from dask.distributed import Client
egg_file = Path(
"__file__"
).parent / "dist" / "dask_cloudprovider_problem-0.0.0-py3.7.egg"
assert egg_file.exists()
client = Client(FargateCluster(n_workers=1))
client.upload_file(egg_file)
dask_cloudprovider_bug_minimal_example.run_tasks()
import scipy.ndimage as ndi
import scipy.sparse as sp
from scipy.interpolate import interp1d
from skimage import filters
# Most relevant functions can be found in registration
from registration import *
plt.rcParams["figure.figsize"] = [12., 8.]
SAVEFIG = True
# -
cluster = LocalCluster(n_workers=1, threads_per_worker=8)
client = Client(cluster)
client.upload_file('registration.py')
client
def plot_stack(images, n, grid=False):
"""Plot the n-th image from a stack of n images.
For interactive use with ipython widgets"""
im = images[n, :, :].compute()
plt.figure(figsize=[12,10])
plt.imshow(im.T, cmap='gray', vmax=im.max())
if grid:
plt.grid()
plt.show()
# +
# A bunch of constants
parser.add_argument("experiment",
type=str,
help="Name of the experiment being performed")
parser.add_argument("iterations", type=int, help="number of iterations")
parser.add_argument("--benchmark",
action="store_true",
help="benchmark results")
args = parser.parse_args()
# Cluster scheduler
cluster = args.scheduler
client = Client(cluster)
print(client)
client.upload_file(
"nfs/SOEN-499-Project/utils.py") # Allow workers to use module
client.upload_file("nfs/SOEN-499-Project/kmeans/Kmeans.py")
# Read images
paths = crawl_dir(os.path.abspath("test/data"))
img = [read_img(path, start=start, args=args) for path in paths]
voxels = da.concatenate([x[1] for x in img]).reshape(-1)
centroids = [0.0, 125.8, 251.6, 377.4] # Initial centroids
voxel_pair = None
bincount = da.bincount(voxels)
bincount = bincount[bincount != 0]
unique = da.unique(voxels)
from bokeh.plotting import figure
from bokeh.io import show, output_notebook
from dask.distributed import Client
import dask.delayed
import allen_comparison
import time
#output_notebook()
directory = "/home/jdehning/tmp/Emx1-s_highzoom"
#directory = "/scratch.local/jdehning/calcium_ephys_comparison_data/processed_data/Emx1-s_highzoom"
#directory = "/scratch.local/jdehning/calcium_ephys_comparison_data/processed_data/Emx1-s_lowzoom"
#client = Client('localhost:8786')
client = Client('localhost:42747')
client.upload_file('allen_comparison.py')
client.run(importlib.import_module, 'allen_comparison')
futures = []
last_pos = 0
def modify_doc(doc):
# Set up data
#ephys, ophys, dt = allen_comparison.open_dir(directory)
ephys = allen_comparison.open_ephys(directory, client)
ophys = allen_comparison.open_ophys(directory, client)
k_arr = np.arange(1, 35)
sources = []
plots = []
for i in range(len(ephys)):
source1 = ColumnDataSource(data=dict(x=k_arr, y=np.zeros_like(k_arr)))
parser.add_argument("delay",
type=float,
help="sleep delay during "
"incrementation")
parser.add_argument("--benchmark",
action="store_true",
help="benchmark results")
args = parser.parse_args()
# Cluster scheduler
cluster = args.scheduler
client = Client(cluster)
print(client)
client.upload_file(
"/nfs/paper-big-data-engines/utils.py") # Allow workers to use module
client.upload_file(
"/nfs/paper-big-data-engines/incrementation/Increment.py")
# Read images
paths = crawl_dir(os.path.abspath(args.bb_dir))
client.scatter(paths)
results = []
for path in paths:
img = client.submit(read_img, path, start=start, args=args)
# Increment the data n time:
for _ in range(args.iterations):
img = client.submit(increment,
img,
parser.add_argument(
"bb_dir",
type=str,
help=("The folder containing BigBrain NIfTI images" "(local fs only)"),
)
parser.add_argument(
"output_dir",
type=str,
help=("the folder to save incremented images to" "(local fs only)"),
)
parser.add_argument(
"experiment", type=str, help="Name of the experiment being performed"
)
parser.add_argument("iterations", type=int, help="number of iterations")
parser.add_argument(
"delay", type=float, help="sleep delay during " "incrementation"
)
parser.add_argument("--benchmark", action="store_true", help="benchmark results")
args = parser.parse_args()
# Cluster scheduler
cluster = args.scheduler
client = Client(cluster)
print(client)
client.upload_file("utils.py") # Allow workers to use module
client.upload_file("Increment.py")
configParallel['backend'] = "dask"
error = 0
if 'scheduler' not in args:
logger.error("Backend: %s. Falta scheduler '-x' o '--scheduler'.")
error += 1
if 'package' not in args:
logger.error("Backend: %s. Falta package '-p' o '--package'.")
error += 1
if error:
logger.error("Backend: %s. Hubo %d errores. Saliendo." %
(args.backend, error))
exit(1)
client = Client('tcp://%s:8786' % args.scheduler)
for egg in args.package:
client.upload_file(egg)
configParallel['scheduler_host'] = (args.scheduler, 8786)
elif args.backend == 'daskyarn':
configParallel['backend'] = "dask"
error = 0
if 'package' not in args:
logger.error("Backend: %s. Falta package '-p' o '--package'.")
error += 1
if error:
logger.error("Backend: %s. Hubo %d errores. Saliendo." %
(args.backend, error))
exit(1)
else:
pass
# Carga datos
