def test_apply(self):
base_row = [1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9]
code_row = 'lambda row : [{}]'\
.format(','.join(['row[{}]+2'.format(i) for i, _ in enumerate(base_row)]))
f_row = eval(code_row)
f = lambda x: x + 2
N = 2000000
row = list(base_row)
with Timer() as t:
for i in six.moves.range(N):
row = f_row(row)
print(t.elapsed, row)
row = list(base_row)
with Timer() as t:
for i in six.moves.range(N):
row = map(f, row)
print(t.elapsed, row)
def _calculate_score(self, ds, tuple_size):
ss = AzharStochasticSearchMapping(
tuple_size,
int(ds.entry_size / tuple_size),
self.recognized_weight,
self.misclassified_weight,
self.rejected_weight,
self.learning_rate,
)
with Timer(factor=self.factor) as creation:
mapping, gen = ss.run(ds.X_train, ds.y_train)
wsd = WiSARD(mapping)
with Timer(factor=self.factor) as training:
wsd.fit(ds.X_train, ds.y_train)
with Timer(factor=self.factor) as classification:
score = wsd.score(ds.X_test, ds.y_test)
return {
"n": tuple_size,
"accuracy": score,
"creation_time": creation.elapsed,
"training_time": training.elapsed,
"classification_time": classification.elapsed,
"generations": gen,
}
def generate_random_data(record_count, query_count):
"""Generate records/queries random data.
:param record_count: Number of random records to generate
:type record_count: int
:param query_count: Number of random queries to generate
:type query_count: int
:returns: Random records and queries
:type: tuple(list(dict(str)), list(str))
"""
words_per_query = 3
fake = Factory.create()
logging.debug('Generating %d random log records...', record_count)
with Timer() as records_timer:
records = [{
'timestamp': fake.iso8601(),
'message': fake.text()
} for _ in range(record_count)]
logging.debug('Generating log records took %f seconds',
records_timer.elapsed)
logging.debug('Generating %d query arguments...', query_count)
with Timer() as queries_timer:
queries = [
' '.join(fake.words(nb=words_per_query))
for _ in range(query_count)
]
logging.debug('Generating query arguments took %f seconds',
queries_timer.elapsed)
return records, queries
def _calculate_score(self, ds, tuple_size):
with Timer(factor=self.factor) as creation:
ss = AzharParticleSwarmMapping(
tuple_size=tuple_size,
final_number_of_tuples=int(ds.entry_size / tuple_size),
inertia_weight=self.inertia_weight,
recognized_weight=self.recognized_weight,
misclassified_weight=self.misclassified_weight,
rejected_weight=self.rejected_weight,
learning_rate=self.learning_rate,
criticality_limit=3,
)
mapping, gen = ss.create_mapping(ds)
wsd = WiSARD(mapping)
with Timer(factor=self.factor) as training:
wsd.fit(ds.X_train, ds.y_train)
with Timer(factor=self.factor) as classification:
score = wsd.score(ds.X_test, ds.y_test)
return {
"n": tuple_size,
"accuracy": score,
"creation_time": creation.elapsed,
"training_time": training.elapsed,
"classification_time": classification.elapsed,
"generations": gen,
}
def _calculate_score(self, ds, tuple_size):
with Timer(factor=self.factor) as creation:
ga = GuarisaGeneticAlgorithm(
tuple_size=tuple_size,
entry_size=ds.entry_size,
population_size=self.population_size,
theta=0.8,
num_exec=int(self.population_size / 2),
lag=self.lag,
max_ittr=100,
validation_size=0.3,
)
mappings, gen = ga.run(ds.X_train, ds.y_train)
wsd = WiSARD(mappings[0])
with Timer(factor=self.factor) as training:
wsd.fit(ds.X_train, ds.y_train)
with Timer(factor=self.factor) as classification:
score = wsd.score(ds.X_test, ds.y_test)
return {
"n": tuple_size,
"accuracy": score,
"creation_time": creation.elapsed,
"training_time": training.elapsed,
"classification_time": classification.elapsed,
"generations": gen,
}
def test_index_save_load():
data = np.random.uniform(0, 100, size=(500000, 3)).astype(np.float32)
queries = np.random.uniform(0, 100, size=(100, 3)).astype(np.float32)
# Lets create an index of kd-tree
kdtree = pynanoflann.KDTree()
with Timer() as index_build_time:
kdtree.fit(data)
dist1, idx1 = kdtree.kneighbors(queries)
# Save the built index
# NOTE: Only the index will be saved, data points are NOT stored in the index
index_path = '/tmp/index.bin'
try:
os.remove(index_path)
except OSError:
pass
kdtree.save_index(index_path)
assert os.path.exists(index_path)
# Now, load a prebuilt index
# BEWARE, data points must be the same
new_kdtree = pynanoflann.KDTree()
with Timer() as index_load_time:
new_kdtree.fit(data, index_path)
# Fitting with a prebuilt index is much faster, since it only requires loading a binary file
assert index_build_time.elapsed > 10 * index_load_time.elapsed
# At the same time, the results are identical
dist2, idx2 = kdtree.kneighbors(queries)
assert (dist2 == dist1).all()
assert (idx1 == idx2).all()
def _calculate_score(self, ds, tuple_size):
with Timer(factor=self.factor) as creation:
mapping = {}
for label, choice_priorities in self.priorities.items():
mapping[str(label)] = create_mapping_by_restrictions(
self.restrictions[label],
choice_priorities_to_priority_of_choice(choice_priorities),
tuple_size,
ds.entry_size,
)
wsd = wp.Wisard(tuple_size, mapping=mapping)
with Timer(factor=self.factor) as training:
wsd.train(ds.train)
with Timer(factor=self.factor) as classification:
score = wsd.score(ds.test)
return {
"n": tuple_size,
"accuracy": score,
"creation_time": creation.elapsed,
"training_time": training.elapsed,
"classification_time": classification.elapsed,
}
def time(N, M, K, problem, method):
if problem == 'matrix_mult':
A, B, C = init(N, M, K)
with Timer() as t:
if method == 'blas':
matrix_mult_d_blas(A, B, C)
elif method == 'eigen':
matrix_mult_d(A, B, C)
elif method == 'numpy':
C = A @ B
print("%s time: %s" % (method, t.elapsed))
elif problem == 'cholesky_dense':
A, _, C = init(N, N, N)
L = np.tril(A)
D = np.diag(np.random.rand(N))
A = L @ D @ L.T + np.eye(N) # Avoid non-pos def errors.
A = (A + A.T) / 2
with Timer() as t:
if method == 'eigen':
cholesky_dense_d(A, C)
elif method == 'numpy':
C = np.linalg.cholesky(A)
print("%s time: %s" % (method, t.elapsed))
return C
def elasticsearch(host, documents, queries):
"""Index documents and run search queries in elasticsarch.
:param host: Elasticsearch server location
:type count: str
:param documents: Documents to be inserted
:type documents: list(dict(str))
:type queries: Queries to execute
:param queries: list(str)
:returns: Insert and query timers
:rtype: dict(str, contexttimer.Timer)
"""
index_name = 'index'
document_type = 'log'
logging.debug('Connecting to elasticsearch in: %r', host)
es = Elasticsearch(hosts=[host])
es.indices.delete(index=index_name, ignore=404)
es.indices.create(index=index_name)
logging.debug('Indexing %d documents...', len(documents))
actions = [{
'_op_type': 'index',
'_index': index_name,
'_type': document_type,
'_source': document,
} for document in documents]
with Timer() as index_timer:
bulk(es, actions, refresh=True)
logging.debug('Indexing took %f seconds', index_timer.elapsed)
logging.debug('Running %d random search queries...', len(queries))
with Timer() as query_timer:
for query in queries:
body = {
'query': {
'match': {
'message': query
},
},
'size': 1,
'highlight': {
'fields': {
'message': {},
},
},
}
result = es.search(
index=index_name,
doc_type=document_type,
body=body,
)
total = result['hits']['total']
logging.debug('%r -> %d hits', query, total)
if total > 0:
logging.debug(pformat(result['hits']['hits'][0]['highlight']))
logging.debug('Querying took %f seconds', query_timer.elapsed)
return {'index': index_timer, 'query': query_timer}
def main():
start_time = time.time()
with Timer() as timer:
args = build_argparser().parse_args()
stt = DeepSpeechPipeline(
model=args.model,
lm=args.lm,
alphabet=args.alphabet,
beam_width=args.beam_width,
alpha=args.alpha,
beta=args.beta,
device=args.device,
ie_extensions=[(args.device, args.cpu_extension)]
if args.device == 'CPU' else [],
)
wave_read = wave.open(args.input, 'rb')
channel_num, sample_width, sampling_rate, pcm_length, compression_type, _ = wave_read.getparams(
)
assert sample_width == 2, "Only 16-bit WAV PCM supported"
assert compression_type == 'NONE', "Only linear PCM WAV files supported"
assert channel_num == 1, "Only mono WAV PCM supported"
audio = np.frombuffer(wave_read.readframes(pcm_length * channel_num),
dtype=np.int16).reshape(
(pcm_length, channel_num))
wave_read.close()
print(
"Loading, including network weights, IE initialization, LM, building LM vocabulary trie, loading audio: {} s"
.format(timer.elapsed))
print("Audio file length: {} s".format(audio.shape[0] / sampling_rate))
# Now it is enough to call:
# transcription = stt.recognize_audio(audio, sampling_rate)
# if you don't need to access intermediate features like character probabilities or audio features.
with Timer() as timer:
audio_features = stt.extract_mfcc(audio, sampling_rate=sampling_rate)
print("MFCC time: {} s".format(timer.elapsed))
with Timer() as timer:
character_probs = stt.extract_per_frame_probs(audio_features,
wrap_iterator=tqdm)
print("RNN time: {} s".format(timer.elapsed))
with Timer() as timer:
transcription = stt.decode_probs(character_probs)
print("Beam search time: {} s".format(timer.elapsed))
print("Overall time: {} s".format(time.time() - start_time))
print("\nTranscription and confidence score:")
max_candidates = 1
for candidate in transcription[:max_candidates]:
print("{}\t{}".format(
candidate['conf'],
candidate['text'],
))
def test_add_small(SEARCH_TYPE, small_matrix):
print('\nAdd small (%s):' % SEARCH_TYPE)
M = small_matrix['M']
indexer = small_matrix['indexer']
true = {}
true['CSR'] = np.array([2.45, 1.22, 1.74, 0.87, 0.6, 3.93])
true['CSC'] = np.array([1.22, 1.74, 0.6, 3.93, 2.45, 0.87])
for key in M:
print('\n%s matrix' % key)
M_copy_cy = M[key].copy()
M_copy_py = M[key].copy()
with Timer() as t:
if SEARCH_TYPE == 'sorted':
if key == 'CSR':
# Sort indices according to row first
sort_idx = np.lexsort((indexer['col'], indexer['row']))
else:
# Sort indices according to col first
sort_idx = np.lexsort((indexer['row'], indexer['col']))
else:
# Technically don;t need to sort with binary search.
sort_idx = np.arange(indexer['row'].size)
unsort_idx = np.argsort(sort_idx)
print('\tLexsort time: %s' % t.elapsed)
with Timer() as t:
start = t.elapsed
csindexer.apply(M_copy_cy,
indexer['row'][sort_idx],
indexer['col'][sort_idx],
indexer['data'][sort_idx],
'add',
SEARCH_TYPE,
N_THREADS,
True)
print('\tCython function time: %s' % (t.elapsed - start))
print('\tCython time to add: %s' % t.elapsed)
with Timer() as t:
idx_coo = sp.sparse.coo_matrix((indexer['data'], (indexer['row'], indexer['col'])))
M_copy_py += idx_coo
print('\tPython/scipy time to add: %s' % t.elapsed)
assert(np.all((M_copy_cy.data - true[key])**2 < 1e-6))
assert(np.all((M_copy_py.data - true[key])**2 < 1e-6))
def test_get_small(SEARCH_TYPE, small_matrix):
print('\nGet small (%s):' % SEARCH_TYPE)
M = small_matrix['M']
indexer = small_matrix['indexer']
true = np.array([0.45, 0.45, 0.22, 0.74, 0.93, 0.93, 0.93])
for key in M:
print('\n%s matrix' % key)
with Timer() as t:
if SEARCH_TYPE == 'sorted':
if key == 'CSR':
# Sort indices according to row first
sort_idx = np.lexsort((indexer['col'], indexer['row']))
else:
# Sort indices according to col first
sort_idx = np.lexsort((indexer['row'], indexer['col']))
else:
# Technically don;t need to sort with binary search.
sort_idx = np.arange(indexer['row'].size)
unsort_idx = np.argsort(sort_idx)
print('\tLexsort time: %s' % t.elapsed)
with Timer() as t:
data_cy = np.empty(indexer['row'].size, dtype=np.float64)
start = t.elapsed
csindexer.apply(M[key],
indexer['row'][sort_idx],
indexer['col'][sort_idx],
data_cy,
'get',
SEARCH_TYPE,
N_THREADS,
True)
print('\tCython function time: %s' % (t.elapsed - start))
# Unsort data_cy
data_cy = data_cy[unsort_idx]
print('\tCython time to get: %s' % t.elapsed)
with Timer() as t:
data_py = np.squeeze(np.array(M[key][indexer['row'][sort_idx],
indexer['col'][sort_idx]]))
data_py = data_py[unsort_idx]
print('\tPython time to get: %s' % t.elapsed)
assert(np.all((data_cy - true)**2 < 1e-6))
assert(np.all((data_py - true)**2 < 1e-6))
def test(search_type='knn', data_dim=3, n_index_points=2000, n_query_points=100, n_neighbors=10, metric='l2', output=False, radius=1):
data = np.random.uniform(0, 100, size=(n_index_points, data_dim)).astype(np.float32)
queries = np.random.uniform(0, 100, size=(n_query_points, data_dim)).astype(np.float32)
with Timer() as sk_init:
nn = neighbors.NearestNeighbors(n_neighbors=n_neighbors, algorithm='auto', metric=metric, radius=radius)
nn.fit(data)
with Timer() as sk_query:
if search_type == 'knn':
sk_res_dist, sk_res_idx = nn.kneighbors(queries)
else:
sk_res_dist, sk_res_idx = nn.radius_neighbors(queries)
with Timer() as kd_init:
nn = pynanoflann.KDTree(n_neighbors=n_neighbors, metric=metric, radius=radius)
nn.fit(data)
with Timer() as kd_query:
if search_type == 'knn':
kd_res_dist, kd_res_idx = nn.kneighbors(queries)
else:
kd_res_dist, kd_res_idx = nn.radius_neighbors(queries)
# allow small diff due to floating point computation
params = {}
for k in inspect.signature(test).parameters:
params[k] = locals().get(k)
if search_type == 'knn':
assert (kd_res_idx == sk_res_idx).mean() > 0.99, params
assert np.allclose(kd_res_dist, sk_res_dist), params
else:
# sklearn radius search does not allow to return sorted indices
# So let's check as an unordered sets
for k, s in zip(kd_res_idx, sk_res_idx):
if len(k):
rat = len(set(k).intersection(set(s))) / len(k)
assert rat > 0.99
else:
assert (k == s).all()
if output and search_type == 'knn':
diff = kd_res_dist - sk_res_dist
data = [['sk', sk_init, sk_query], ['kd', kd_init, kd_query]]
t = tabulate.tabulate(data, headers=['', 'Init', 'Query'], tablefmt='psql')
print(t)
print('Dist diff: {}'.format(diff.sum()))
print('IDX diff: {} / {}'.format((kd_res_idx != sk_res_idx).sum(), kd_res_idx.size))
def collect(egoids,
args,
exp_dir,
use_hgail,
params_filename,
n_proc,
collect_fn=parallel_collect_trajectories,
random_seed=None,
lbd=0.99,
adapt_steps=1):
'''
Description:
- prepare for running collection in parallel
- multiagent note: egoids and starts are not currently used when running
this with args.env_multiagent == True
'''
# load information relevant to the experiment
params_filepath = os.path.join(exp_dir,
'imitate/{}'.format(params_filename))
params = np.load(params_filepath)['params'].item()
# validation setup
validation_dir = os.path.join(exp_dir, 'imitate', 'test')
utils.maybe_mkdir(validation_dir)
with Timer():
error = collect_fn(args,
params,
egoids,
n_proc,
use_hgail=use_hgail,
random_seed=random_seed,
lbd=lbd,
adapt_steps=adapt_steps)
return error
def run_single_simulation(
cfg,
verbose=False,
force_rerun=False,
only_initialize_network=False,
save_initial_network=False,
save_csv=False,
) :
with Timer() as t, warnings.catch_warnings() :
if not verbose :
# ignore warning about run_algo
warnings.simplefilter("ignore", NumbaExperimentalFeatureWarning)
warnings.simplefilter("ignore", NumbaTypeSafetyWarning)
# warnings.simplefilter("ignore", NumbaPendingDeprecationWarning)
simulation = Simulation(cfg, verbose)
simulation.initialize_network(
force_rerun=force_rerun, save_initial_network=save_initial_network, only_initialize_network=only_initialize_network
)
if only_initialize_network :
return None
simulation.initialize_states()
simulation.run_simulation()
simulation.save(time_elapsed=t.elapsed, save_hdf5=True, save_csv=save_csv)
return cfg
def import_(
input_file,
source_language,
target_language,
database_url,
chunk_size,
min_entries,
confirm,
):
"""Import new entries into the dictionary database."""
filename = input_file.name or "<stdin>"
logger.info(f'Starting dictionary import from file "{filename}"…')
if confirm:
confirm_or_exit("This will remove all existing entries. Continue?")
engine = prepare_engine(database_url)
try:
with Timer() as timer:
entries = load_entries(input_file)
num_added, num_deleted = import_entries(
engine,
entries,
source_language,
target_language,
chunk_size=chunk_size,
min_entries=min_entries,
)
except Exception as exc:
logger.exception(f"Failed to import entries: {exc!r}")
sys.exit(errno.EIO)
logger.info(
f"Successfully completed dictionary import ({num_deleted} deleted, "
f"{num_added} added, {format_timespan(timer.elapsed)} elapsed).")
def optimal_policy(env, return_qvalue=False, verbose=False):
"""
Compute the optimal policy for environment 'env'.
Parameters
----------
env : Trial
return_qvalue : bool
verbose : bool
Returns
-------
policy : FunctionPolicy
Q : function( [ Categorical/Normal/int ], int )
"""
with Timer() as t:
Q, V, pi, info = solve(env)
v = V(env.init)
if verbose:
print('optimal -> {:.2f} in {:.3f} sec'.format(v, t.elapsed))
policy = FunctionPolicy(pi)
if return_qvalue:
return policy, Q
else:
return policy
def restore(url, dumpfile):
ctx = click.get_current_context()
from_db = ctx.params["from_db"]
if from_db.scheme == "postgresql":
dbname = from_db.path[1:]
else:
dbname = parse_custom_scheme(from_db)["database"]
common = "--clean --no-acl --no-owner"
if url.scheme == "postgresql":
create_cmd = f"createdb --dbname -T template0 {url.geturl()} {dbname}"
restore_cmd = f"pg_restore --dbname {url.geturl()} {common}"
else:
d = parse_custom_scheme(url)
create_cmd = d[
"base"] + f" createdb -T template0 -U {d['username']} {dbname}"
restore_cmd = d[
"base"] + f" pg_restore {common} -U {d['username']} -d {dbname}"
create_result = subprocess.run(create_cmd.split(),
check=False,
capture_output=True,
encoding="utf8")
if create_result.returncode != 0 and "already exists" not in create_result.stderr:
click.secho(create_result.stderr, fg="red")
raise click.Abort()
with open(dumpfile, "r") as f:
with Halo("Restoring...") as h, Timer() as t:
subprocess.run(restore_cmd.split(), check=True, stdin=f)
h.succeed()
click.secho(f"Restore took {t.elapsed:.2f} seconds", fg="green")
def main():
X_all = np.load('./cache/composer/raw.npy', allow_pickle=False)
print(X_all.dtype, X_all.shape)
saver = tf.train.Saver()
with Timer() as t:
with tf.Session() as sess:
init.run()
try:
for epoc in range(n_epochs):
for iteration in range(n_iterations):
X_batch, y_batch = next_batch(X_all, batch_size,
config.n_steps)
sess.run(training_op,
feed_dict={X: X_batch, y: y_batch})
if iteration % 10 == 0:
loss_eval = loss.eval(
feed_dict={X: X_batch, y: y_batch})
print(iteration, "t = ", t.elapsed, "Loss: ",
loss_eval)
saver.save(sess, './save/composer')
if loss_eval < 0.001:
raise EarlyTermination
if t.elapsed > 2500:
raise EarlyTermination
except EarlyTermination:
pass
def get_c2():
print("Regenerating 2-qubit Clifford group...")
with Timer() as t:
c2 = make_clifford_group(2,
get_clifford_2q_xzpm2_cz_implementation)
print("...done ({:.3f} s).".format(t.elapsed))
return c2
def loadAllFeatureSets(platform, seed, fraction, include_crowd_data, use_processes, exclude_labels, do_print=True):
all_sets = []
filenames = glob.glob('./data/*.fsets.{}.pickle'.format(platform))
if fraction < 1.0:
random.seed(seed)
print "Training on a random subset of feature sets: {:.2f}".format(fraction)
filenames = random.sample(filenames, max(1, int(len(filenames) * fraction)))
with Timer() as t:
for filename in filenames:
sets = loadFeatureSets(filename)
if len(sets) > 0 and sets[0].reportedActivityType not in exclude_labels:
all_sets += sets
print "Loaded {} labeled feature sets from {} files in {:.1f}s".format(len(all_sets), len(filenames), t.elapsed)
if include_crowd_data:
config = all_sets[0].forestConfigStr
print "Loading whitelisted TSDs"
all_sets += getFeatureSetsFromAllTrainableTSDs(platform, config, use_processes=use_processes)
print "Loading trusted TSD events"
all_sets += getFeatureSetsFromTrustedEventsPickle(platform, config)
return all_sets
def test_wait(mock_server):
imposter = Imposter(Stub(responses=Response(wait=100)))
with mock_server(imposter), Timer() as timer:
requests.get(imposter.url)
assert_that(timer.elapsed, between(0.1, 0.25))
def multi_core_benchmark(n: int, core_config: tp.Iterable[int], repetitions: int = 1) \
-> tp.Dict[int, float]:
# This does not increase performance on Anaconda and Windows as elementwise trigonometric
# operations seem to be multi-threaded to begin with.
number_of_cores_to_runtime_map = {}
a_complete, b_complete, c_complete = generate_data(n, gpu=False)
for number_of_cores in core_config:
with Timer() as timer:
for _ in range(repetitions):
kwargs_list = split_arrays(a=a_complete,
b=b_complete,
c=c_complete,
number_of_batches=number_of_cores)
result = \
map_reduce_multicore(f=lambda a, b, c: process_data(a, b, c, gpu=False),
reduction=lambda x, y: numpy.hstack((x, y)),
kwargs_list=kwargs_list)
assert isinstance(result, numpy.ndarray)
assert result.shape == (n, )
number_of_cores_to_runtime_map[
number_of_cores] = timer.elapsed / repetitions
return number_of_cores_to_runtime_map
def train(self):
"""Train the regression model."""
# Load data.
with Timer() as t:
if self.method == 'true':
X = np.load(self.data_dir + '/train/X.npy')
elif self.method == 'mean':
X = np.load(self.data_dir + '/train/mu.npy')
elif self.method == 'sample':
mu = np.load(self.data_dir + '/train/mu.npy')
L_T = np.load(self.data_dir + '/train/L_T.npy')
X = np.array([
sp.sparse.linalg.spsolve_triangular(
t,
np.random.randn(self.R).astype(np.float32),
lower=False) for t in L_T
]).astype(np.float32)
X = X + mu
if self.config['debug']:
print("Time to load/sample data: %s." % t.elapsed)
Y = np.load(self.data_dir + '/train/Y.npy')
self.lr.fit(X, Y)
self.episode_id += 1
if self.config['debug']:
# Print the mean log-likelihood on the data.
pred = self.infer(X)
LL = np.mean(np.log(pred[np.arange(Y.size), Y]))
print('Log likelihood on training data: %s' % LL)
def run(self, ds):
print("=== INFO ===\nExperimento: {}\nDataset: {}".format(
self.experiment_name, ds.get_name()))
with Timer(factor=self.factor) as t:
self._prep(ds)
prep_time = t.elapsed
filename = self.get_filename(ds.dataset_name, ds.binarization_name)
df = self.load(filename)
tuple_sizes = valid_tuple_sizes(ds.entry_size)
print("Tuple sizes: ", tuple_sizes)
for tuple_size in tuple_sizes:
nec_exec = self.num_exec - len(df.loc[df["n"] == tuple_size])
for i in range(nec_exec):
print("-- {} ({}/{})".format(tuple_size, i + 1, nec_exec),
end="\r")
mapping_score = self._calculate_score(ds, tuple_size)
mapping_score["time_prep"] = prep_time
df = df.append(
mapping_score,
ignore_index=True,
)
print("-- {}\tOK! ".format(tuple_size))
if self.save:
df.to_csv(filename, index=False)
print("=============")
return df
def run(tn=4000, space_order=4, kernel='OT4', nbpml=40, tolerance=0.01, parallel_compression=True, filename='', **kwargs):
if kernel in ['OT2', 'OT4']:
solver = overthrust_setup(filename=filename, tn=tn, nbpml=nbpml, space_order=space_order, kernel=kernel, **kwargs)
elif kernel == 'TTI':
solver = overthrust_setup_tti(filename=filename, tn=tn, nbpml=nbpml, space_order=space_order, kernel=kernel, **kwargs)
else:
raise ValueError()
total_timesteps = solver.source.time_range.num
u = None
rec = None
results = []
print(total_timesteps)
for t in range(1, total_timesteps-1):
return_values = solver.forward(u=u, rec=rec, time_m=t, time_M=t, save=False)
rec = return_values[0]
last_time_step = rec.shape[0] - 1
u = return_values[1]
uncompressed = u.data[t+1]
print(np.linalg.norm(uncompressed))
with Timer(factor=1000) as time1:
compressed = compress(uncompressed, tolerance=tolerance, parallel=parallel_compression)
result = (t, len(uncompressed.tostring())/float(len(compressed)), time1.elapsed)
results.append(result)
print(result)
ret = solver.forward(save=False)
assert(ret[1].shape == u.shape)
assert(np.all(np.isclose(ret[1].data, u.data)))
with open('results.csv', 'w+') as csvfile:
writer = csv.writer(csvfile)
for row in results:
writer.writerow(row)
def raw_text(input_file: str, output_file: str = None, gpu: bool = False, time: bool = True, memory: bool = True,
ner: bool = True, model_name: str = "hu_core_news_lg"):
nlp = load_pipeline(gpu, ner, model_name)
if output_file:
nlp.add_pipe("conll_formatter")
data_file = open(input_file, "r", encoding="utf-8")
sentences = list(parse_incr(data_file))
texts = " ".join([s.metadata["text"] for s in sentences])
if time:
with Timer() as t:
res = nlp(texts)
print(f'Time spent: {t.elapsed:.2f} seconds')
else:
res = nlp(texts)
if output_file:
with open(output_file, 'w', encoding='utf-8') as writer:
# noinspection PyProtectedMember
print(rename_root(res._.conll_str), sep="\n", file=writer)
if memory:
print(f'Maximum memory usage: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024:.2f} MiB')
def single_gpu_benchmark(n: int, batches: int, repetitions: int = 1) -> float:
with Timer() as timer:
for _ in range(repetitions):
estimate_pi(n, batches=batches, gpu=True)
sync()
return timer.elapsed / repetitions
def profile():
N = 10000
scale = N**(1/2)
ps = np.random.uniform(-1*scale, 1*scale, (N, 2))
with Timer() as t:
labels = dbscan(ps, 3, 0.5)
print("elapesed: {:.4f}".format(t.elapsed), "num labels:", len(set(labels)))
def multi_gpu_benchmark(n: int,
batches: int,
compute_device_pool: ComputeDevicePool,
repetitions: int = 1,
verbose: bool = False) -> tp.Dict[int, float]:
if verbose:
print('multi gpu benchmark - begin')
number_of_devices_to_runtime_map = {}
for number_of_devices_to_use in range(1, compute_device_pool.number_of_devices + 1):
with Timer() as timer:
for _ in range(repetitions):
pi = compute_device_pool.map_reduce(lambda: estimate_pi(n=math.ceil(n / number_of_devices_to_use), batches=batches, gpu=True),
reduction=lambda x, y: x + y / number_of_devices_to_use,
initial_value=0.0,
number_of_batches=number_of_devices_to_use)
sync()
if verbose:
print(pi)
gpu_time = timer.elapsed / repetitions
number_of_devices_to_runtime_map[number_of_devices_to_use] = gpu_time
if verbose:
print('multi gpu benchmark - end')
return number_of_devices_to_runtime_map
