def RunWithLogging(numberOfRepetitions, players = None, saveGameStateLogs = False, agentIndex = 0, multiprocess = False):
if players is None:
players = defaultPlayers
logger = logging.getLogger()
today = datetime.datetime.today()
fileName = 'SimulatorLogs/log_{0}.txt'.format(today.strftime("%d-%m-%Y_%H-%M"))
fileNameSimulations = 'SimulatorLogs/logSim_{0}.txt'.format(today.strftime("%d-%m-%Y_%H-%M"))
logFile = logging.FileHandler(fileName)
logger.addHandler(logFile)
winCount = [0, 0, 0, 0]
totalTime = datetime.datetime.utcnow()
if multiprocess:
num_cores = multiprocessing.cpu_count()
winners = Parallel(n_jobs=num_cores)(delayed(RunParallel)
(CreateGame(players), i, numberOfRepetitions, fileNameSimulations)
for i in range(0, numberOfRepetitions))
winCount = [winners.count(0), winners.count(1), winners.count(2), winners.count(3)]
else:
games = [CreateGame(players) for i in range(numberOfRepetitions)]
for i in range(0, numberOfRepetitions):
time = datetime.datetime.utcnow()
winner = RunGame(games[i], games[i].gameState.players, showLog=True, showFullLog=False, saveImgLog=saveGameStateLogs)
logging.critical("\n GAME TIME = {0}".format(((datetime.datetime.utcnow() - time).total_seconds())))
winCount[winner] += 1
total = "\n TOTAL GAMES = {0}/{1} ".format(
(i + 1),
numberOfRepetitions)
print(total)
logging.critical(total)
logging.critical("\n TOTAL TIME = {0}".format(((datetime.datetime.utcnow() - totalTime).total_seconds())/60.0))
logging.critical(" TOTAL GAMES = {0} \n WIN COUNT = {1} \n AGENT WIN PERCENTAGE = {2}%".format(
numberOfRepetitions,
winCount,
(float(winCount[agentIndex]) / numberOfRepetitions) * 100.0
))
def p_w_l(l):
r = []
for i in range(0, l):
r.append(['T', 'F', ' '])
ps = [0, 0, 0]
results = Parallel(n_jobs=4)(delayed(p_w)(e) for e in product(*r))
ps[0] = results.count(0)
ps[1] = results.count(1)
ps[2] = results.count(2)
return ps
def refine_thar_positions(wave_init, order_init, thar1d_fixed, thar_list,
fit_width=5., lc_tol=5., k=3, n_jobs=10, verbose=10):
print("@Cham: refine ThAr positions ...")
# refine thar positions for each order
r = Parallel(n_jobs=n_jobs, verbose=verbose, batch_size=1)(
delayed(refine_thar_positions_order)(
wave_init[i_order],
np.arange(wave_init.shape[1]),
thar1d_fixed[i_order],
thar_list[(thar_list > np.min(wave_init[i_order]) + 1.) * (
thar_list < np.max(wave_init[i_order]) - 1.)],
order_init[i_order, 0],
fit_width=fit_width, lc_tol=lc_tol, k=k
) for i_order in range(wave_init.shape[0]))
# remove all null values
null_value = (None, None, None, None, None)
for i in range(r.count(null_value)):
r.remove(null_value)
print(len(r))
# collect data
lc_coord = np.array(np.hstack([_[0] for _ in r]))
lc_order = np.array(np.hstack([_[1] for _ in r]))
lc_thar = np.array(np.hstack([_[2] for _ in r]))
popt = np.array(np.vstack([_[3] for _ in r]))
pcov = np.array(np.vstack([_[4] for _ in r]))
return lc_coord, lc_order, lc_thar, popt, pcov
def upload_retrospective_study(path_study_root_folder_path):
"""
Sample script to recursively import in Orthanc all the DICOM files
that are stored in some path. Please make sure that Orthanc is running
before starting this script. The files are uploaded through the REST
API.
Usage: %s [hostname] [HTTP port] [path]
Usage: %s [hostname] [HTTP port] [path] [username] [password]
For instance: %s 127.0.0.1 8042 .
:param path_study_root_folder_path:
:return: 1
"""
success_count = 0
total_file_count = 0
if os.path.isfile(path_study_root_folder_path):
# Upload a single file
total_file_count = 1
success_count = read_upload_file(path_study_root_folder_path)
else:
# Recursively upload a directory
list_files = recursive_list(path_study_root_folder_path)
total_file_count = len(list_files)
# Serial process them:
# for file in tqdm(list_files):
# if read_upload_file(file):
# success_count += 1
# Parallel process them
num_cores = multiprocessing.cpu_count()
# Store the output in a list
results = Parallel(n_jobs=num_cores)(delayed(read_upload_file)(i)
for i in list_files)
# Count success.
success_count = results.count(True)
if success_count == total_file_count:
logger.info(
"\nSummary: all %d DICOM file(s) have been imported successfully" %
success_count)
else:
logger.warning(
"\nSummary: %d out of %d files have been imported successfully as DICOM instances"
% (success_count, total_file_count))
def test_rf_feats_fullspec(slice_infos, i):
feats, labels = generate_training_feats(slice_infos, i)
RF = train_rf_classifier(feats, labels, no_trees)
test_sl = slice_infos[i]
image = test_sl.slice_im
label = test_sl.slice_lb
patches_m, patches_n = extract_roi_patches(image, label, psize)
patches_n = patches_n
plabels_m = ['M' for m in patches_m]
plabels_n = ['N' for n in patches_n]
kernels = generate_kernels()
tot = len(patches_m) + len(patches_n)
res1 = []
res2 = []
t0 = time()
if len(sys.argv) >= 2:
res1 = Parallel(n_jobs=int(sys.argv[1]))(
delayed(check_classify)(RF, kernels, p, plabels_m[i], i, tot)
for i, p in enumerate(patches_m))
res2 = Parallel(n_jobs=int(sys.argv[1]))(
delayed(check_classify)(RF, kernels, p, plabels_n[i], i, tot)
for i, p in enumerate(patches_n))
else:
for i, p in enumerate(
patches_m): # go through each patch, and classify it!
res1.append(check_classify(RF, kernels, p, plabels_m[i], i, tot))
pass
dt = time() - t0
print(res1.count(True) + res2.count(True))
print("Finished in {:.2f} seconds.".format(dt))
def checkCluster(self, df1, tanimotoDf, threshold):
taniDF = tanimotoDf[tanimotoDf.sum(axis=1) != 0]
clusters = df1['cluster'].unique().tolist()
#startTime=time.time()
r = Parallel(n_jobs=-1, backend="threading")([
delayed(self.checkClusterInner)(cluster, df1, taniDF)
for cluster in clusters
])
print(r)
try:
single = r.count(0)
except:
single = 0
if len(r) == 0:
return 0
else:
return sum(r), single
def save(
self,
root: Union[str, Path],
kind: Optional[str] = "json",
n_jobs: int = 1,
ignore_exceptions: bool = True,
):
"""Save all the music objects to a directory.
The converted files will be named by its index and saved to ``root/``.
Parameters
----------
root : str or Path
Root directory to save the data.
kind : {'json', 'yaml'}, optional
File format to save the data. Defaults to 'json'.
n_jobs : int, optional
Maximum number of concurrently running jobs in multiprocessing. If
equal to 1, disable multiprocessing. Defaults to 1.
ignore_exceptions : bool, optional
Whether to ignore errors and skip failed conversions. This can be
helpful if some of the source files is known to be corrupted.
Defaults to False.
Notes
-----
The original filenames can be found in the ``filenames`` attribute.
For example, the file at ``filenames[i]`` will be converted and
saved to ``{i}.json``.
"""
if kind not in ("json", "yaml"):
raise TypeError("`kind` must be either 'json' or 'yaml'.")
if not isinstance(n_jobs, int):
raise TypeError("`n_jobs` must be an integer.")
if n_jobs < 0:
raise ValueError("`n_jobs` must be positive.")
root = Path(root).expanduser().resolve()
if not root.exists():
raise ValueError("`root` must be an existing path.")
def _saver(idx):
prefix = "0" * (n_digits - len(str(idx)))
if ignore_exceptions:
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
self[idx].save(
root / (prefix + str(idx) + "." + kind), kind
)
except Exception: # pylint: disable=broad-except
return False
return True
self[idx].save(root / (prefix + str(idx) + "." + kind), kind)
return True
n_digits = len(str(len(self)))
print("Start converting and saving the dataset.")
if n_jobs == 1:
count = 0
for idx in tqdm(range(len(self))): # type: ignore
if _saver(idx):
count += 1
else:
if not HAS_JOBLIB:
raise ValueError(
"Optional package joblib is required for multiprocessing "
"(n_jobs > 1)."
)
# TODO: This is slow as `self` is passed between workers.
results = Parallel(n_jobs=n_jobs, backend="threading", verbose=5)(
delayed(_saver)(idx) for idx in range(len(self))
)
count = results.count(True)
print(
"{} out of {} files successfully saved.".format(count, len(self))
)
(root / ".muspy.success").touch(exist_ok=True)
for line in cmd.stderr:
try:
return float(line)
except ValueError:
print(line)
return 0.0
# Compare all compressed images with original video
for g in gop:
for f in format:
for r in fps:
rr = str(eval(r))
for b in bitrate:
folder_images = 'videos_{}gop_{}format_{}fps_{}bitrate_images'.format(g, f, rr, b)
folder_original = 'videos_{}format_{}fps_images'.format(f, rr)
compare_total = 0.0
if os.path.exists('{}/results.txt'.format(folder_images)):
break
for name in os.listdir('./original'):
compare_list = []
compare_list = Parallel(n_jobs=num_cores)(delayed(launch_process)(folder_original, image, name, folder_images) for image in os.listdir('./original_images/{}/{}'.format(folder_original, name)))
file_results = open('{}/results.txt'.format(folder_images), "a")
file_results.write('{}: {}\n'.format(name, sum(compare_list)/(len(compare_list)-compare_list.count(0.0))))
file_results.close()
compare_total += sum(compare_list)/(len(compare_list)-compare_list.count(0.0))
file_results = open('{}/results.txt'.format(folder_images), "a")
file_results.write('\n{}: total {}\n'.format(compare_total, folder_images))
file_results.close()
def save(
self,
root: Union[str, Path],
kind: str = "json",
n_jobs: int = 1,
ignore_exceptions: bool = True,
verbose: bool = True,
**kwargs,
):
"""Save all the music objects to a directory.
Parameters
----------
root : str or Path
Root directory to save the data.
kind : {'json', 'yaml'}, optional
File format to save the data. Defaults to 'json'.
n_jobs : int, optional
Maximum number of concurrently running jobs. If equal to 1,
disable multiprocessing. Defaults to 1.
ignore_exceptions : bool, optional
Whether to ignore errors and skip failed conversions. This
can be helpful if some source files are known to be
corrupted. Defaults to True.
verbose : bool, optional
Whether to be verbose. Defaults to True.
**kwargs
Keyword arguments to pass to :func:`muspy.save`.
"""
if kind not in ("json", "yaml"):
raise TypeError("`kind` must be either 'json' or 'yaml'.")
root = Path(root).expanduser().resolve()
root.mkdir(exist_ok=True)
def _saver(idx):
prefix = "0" * (n_digits - len(str(idx)))
filename = root / (prefix + str(idx) + "." + kind)
if ignore_exceptions:
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
save(filename, self[idx], kind, **kwargs)
except Exception: # pylint: disable=broad-except
return False
return True
save(filename, self[idx], kind, **kwargs)
return True
n_digits = len(str(len(self)))
if verbose:
print("Converting and saving the dataset...")
if n_jobs == 1:
count = 0
for idx in tqdm(range(len(self))): # type: ignore
if _saver(idx):
count += 1
else:
# TODO: This is slow as `self` is passed between workers.
results = Parallel(n_jobs=n_jobs, backend="threading",
verbose=5)(delayed(_saver)(idx)
for idx in range(len(self)))
count = results.count(True)
if verbose:
print(f"Successfully saved {count} out of {len(self)} files.")
xlsnamearr.append(name[:-5])
xlspatharr.append(path)
xlsfilepatharr.append(os.path.join(path, name[:-5]))
xlslmt.append(os.path.getmtime(os.path.join(path, name)))
xlsct.append(os.path.getctime(os.path.join(path, name)))
#convert all to numpy array
csvnamearr=np.array(csvnamearr)
xlsnamearr =np.array(xlsnamearr)
csvpatharr =np.array(csvpatharr)
xlspatharr =np.array(xlspatharr)
csvfilepatharr=np.array(csvfilepatharr)
xlsfilepatharr=np.array(xlsfilepatharr)
csvlmt=np.array(csvlmt)
csvct=np.array(csvct)
xlslmt =np.array(xlslmt)
xlsct=np.array(xlsct)
#call function to write csv files in a loop
#The maximum number of concurrently running jobs, such as the number of Python worker
#processes when backend=”multiprocessing” or the size of the thread-pool when
#backend=”threading”. If -1 all CPUs are used. If 1 is given, no parallel computing code
#is used at all, which is useful for debugging. For n_jobs below -1,
#(n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one are used.
if __name__ == '__main__':
ret=Parallel(n_jobs=-1,verbose=5)(delayed(csvwrite)(x,csvfilepatharr,xlsfilepatharr,xlsnamearr,csvlmt,xlslmt,xlspatharr,xlsfilearr) for x in range(0,len(xlsnamearr)))
print("Number of Excel files: "+ str(len(xlsnamearr))+"\n")
print("Number of Excel files processed: "+ str(ret.count(1))+"\n")
print("Number of Excel files Skipped Since CSV Exists: "+ str(ret.count(0))+"\n")
print("Number of Excel files Skipped Because of Bad Sheets: "+ str(ret.count(2))+"\n")
print("Number of Excel files Not Matching String Criterion: "+ str(ret.count(3))+"\n")
#nmax = 40000 #Número máximo hasta el que queremos buscar primos
nmax = 400000 #Número máximo hasta el que queremos buscar primos
inputs = range(0, nmax)
def isPrime(num):
if num < 1:
return False
elif num == 2:
return True
else:
for i in range(2, num):
if num % i == 0:
return False
return True
starttime = time.time()
num_cores = multiprocessing.cpu_count()
#Si accedemos a zonas de memoria comunes (listas diccionarios...) , require='sharedmem'
#pero el rendimiento cae mucho:
results = Parallel(n_jobs=num_cores )(delayed(isPrime)(i) for i in inputs)
print ('encontrados en si %s' % results.count(True))
print ('encontrados en no %s' % results.count(False))
print('That took {} seconds'.format(time.time() - starttime))
