def compute(task_length):
samples = list()
samples.append(
['cpu', 0, {
'time': task_length,
'flops': 0,
'efficiency': 1
}])
info, ret, out = rs.emulate(samples=samples)
def read_compute_write(input_files, r_bufsize, task_length, output_files,
output_sizes, w_bufsize):
# we chunk the compute according to output chunking
total_size = 0
for output_size in output_sizes:
total_size += output_size
nreads = int(total_size / bufsize)
cpu_chunk = int(task_length / nreads)
samples = list()
n = 0
for input_file, output_file, output_size in zip(input_files, output_files,
output_sizes):
r_size = os.stat(input_file).st_size
w_size = output_size
n_samples = fsize / bufsize
for s in range(n_samples):
n += 2
samples.append([
'sto', t, {
'src': input_file,
'rsize': int(r_size / n_samples),
'tgt': None,
'wsize': 0,
'buf': r_bufsize
}
])
samples.append([
'cpu', t + 1, {
'time': cpu_chunk,
'flops': 0,
'efficiency': 1
}
])
samples.append([
'sto', t + 1, {
'src': None,
'rsize': 0,
'tgt': output_file,
'wsize': int(w_size / n_samples),
'buf': w_bufsize
}
])
info, ret, out = rs.emulate(samples=samples)
def read_compute(input_files, bufsize, task_mode, task_length):
total_size = 0
for input_file in input_files:
total_size += os.stat(input_file).st_size
nreads = int(total_size / bufsize)
cpu_chunk = int(task_length / nreads)
samples = list()
n = 0
for input_file in input_files:
fsize = os.stat(input_file).st_size / bufsize
n_samples = fsize / bufsize
for s in range(n_samples):
# construct artificial time)
n += 1
if task_mode == 'time':
samples.append([
'cpu', t, {
'time': cpu_chunk,
'flops': 0,
'efficiency': 1
}
])
elif task_mode == 'flops':
samples.append([
'cpu', t, {
'time': 0,
'flops': cpu_chunk,
'efficiency': 1
}
])
samples.append([
'sto', t, {
'src': input_file,
'rsize': bufsize,
'tgt': None,
'wsize': 0,
'buf': bufsize
}
])
info, ret, out = rs.emulate(samples=samples)
def writefiles(output_files, output_sizes, bufsize):
samples = list()
n = 0
for output_file, output_size in zip(output_files, output_sizes):
n += 1
samples.append(['sto', n, {'src' : None,
'rsize' : 0,
'tgt' : output_file,
'wsize' : output_size,
'buf' : bufsize}])
info, ret, out = rs.emulate(samples=samples)
def compute_write(task_mode, task_length, output_files, output_sizes, bufsize):
total_size = 0
for output_size in output_sizes:
total_size += output_size
nreads = int(total_size / bufsize)
cpu_chunk = int(task_length / nreads)
samples = list()
n = 0
for output_file, output_size in zip(output_files, output_sizes):
fsize = output_size
n_samples = fsize / bufsize
for s in range(n_samples):
n += 1
if task_mode == 'time':
samples.append([
'cpu', n, {
'time': cpu_chunk,
'flops': 0,
'efficiency': 1
}
])
elif task_mode == 'flops':
samples.append([
'cpu', n, {
'time': 0,
'flops': cpu_chunk,
'efficiency': 1
}
])
samples.append([
'sto', n, {
'src': None,
'rsize': 0,
'tgt': output_size,
'wsize': bufsize,
'buf': bufsize
}
])
info, ret, out = rs.emulate(samples=samples)
def readfiles(input_files, bufsize):
samples = list()
n = 0
for input_file in input_files:
n += 1
fsize = os.stat(input_file).st_size
samples.append(['sto', n, {'src' : input_file,
'rsize' : fsize,
'tgt' : None,
'wsize' : 0,
'buf' : bufsize}])
info, ret, out = rs.emulate(samples=samples)
def compute(task_mode, task_length):
samples = list()
if task_mode == 'time' :
samples.append(['cpu', 0, {'time' : task_length,
'flops' : 0,
'efficiency' : 1}])
elif task_mode == 'flops' :
samples.append(['cpu', 0, {'time' : 0,
'flops' : task_length,
'efficiency' : 1}])
else:
raise ValueError('Unknown task mode %s' % task_mode)
info, ret, out = rs.emulate(samples=samples)
def read_compute_write(input_files, r_bufsize,
task_mode, task_length,
output_files, output_sizes, w_bufsize):
# we chunk the compute according to output chunking
total_size = 0
for output_size in output_sizes:
total_size += output_size
nreads = int(total_size / bufsize)
cpu_chunk = int(task_length / nreads)
samples = list()
n = 0
for input_file, output_file, output_size in zip(input_files, output_files, output_sizes):
r_size = os.stat(input_file).st_size
w_size = output_size
n_samples = fsize / bufsize
for s in range(n_samples):
n += 2
samples.append(['sto', t, {'src' : input_file,
'rsize' : int(r_size/n_samples),
'tgt' : None,
'wsize' : 0,
'buf' : r_bufsize}])
if task_mode == 'time':
samples.append(['cpu', t+1, {'time' : cpu_chunk,
'flops' : 0,
'efficiency' : 1}])
elif task_mode == 'flops':
samples.append(['cpu', t+1, {'time' : 0,
'flops' : cpu_chunk,
'efficiency' : 1}])
samples.append(['sto', t+1, {'src' : None,
'rsize' : 0,
'tgt' : output_file,
'wsize' : int(w_size/n_samples),
'buf' : w_bufsize}])
info, ret, out = rs.emulate(samples=samples)
def writefiles(output_files, output_sizes, bufsize):
samples = list()
n = 0
for output_file, output_size in zip(output_files, output_sizes):
n += 1
samples.append([
'sto', n, {
'src': None,
'rsize': 0,
'tgt': output_file,
'wsize': output_size,
'buf': bufsize
}
])
info, ret, out = rs.emulate(samples=samples)
def readfiles(input_files, bufsize):
samples = list()
n = 0
for input_file in input_files:
n += 1
fsize = os.stat(input_file).st_size
samples.append([
'sto', n, {
'src': input_file,
'rsize': fsize,
'tgt': None,
'wsize': 0,
'buf': bufsize
}
])
info, ret, out = rs.emulate(samples=samples)
def compute(task_mode, task_length):
samples = list()
if task_mode == 'time':
samples.append(
['cpu', 0, {
'time': task_length,
'flops': 0,
'efficiency': 1
}])
elif task_mode == 'flops':
samples.append(
['cpu', 0, {
'time': 0,
'flops': task_length,
'efficiency': 1
}])
else:
raise ValueError('Unknown task mode %s' % task_mode)
info, ret, out = rs.emulate(samples=samples)
def read_compute(input_files, bufsize, task_mode, task_length):
total_size = 0
for input_file in input_files:
total_size += os.stat(input_file).st_size
nreads = int(total_size / bufsize)
cpu_chunk = int(task_length / nreads)
samples = list()
n = 0
for input_file in input_files:
fsize = os.stat(input_file).st_size / bufsize
n_samples = fsize / bufsize
for s in range(n_samples):
# construct artificial time)
n += 1
if task_mode == 'time':
samples.append(['cpu', t, {'time' : cpu_chunk,
'flops' : 0,
'efficiency' : 1}])
elif task_mode == 'flops':
samples.append(['cpu', t, {'time' : 0,
'flops' : cpu_chunk,
'efficiency' : 1}])
samples.append(['sto', t, {'src' : input_file,
'rsize' : bufsize,
'tgt' : None,
'wsize' : 0,
'buf' : bufsize}])
info, ret, out = rs.emulate(samples=samples)
def compute_write(task_mode, task_length, output_files, output_sizes, bufsize):
total_size = 0
for output_size in output_sizes:
total_size += output_size
nreads = int(total_size / bufsize)
cpu_chunk = int(task_length / nreads)
samples = list()
n = 0
for output_file, output_size in zip(output_files, output_sizes):
fsize = output_size
n_samples = fsize / bufsize
for s in range(n_samples):
n += 1
if task_mode == 'time':
samples.append(['cpu', n, {'time' : cpu_chunk,
'flops' : 0,
'efficiency' : 1}])
elif task_mode == 'flops':
samples.append(['cpu', n, {'time' : 0,
'flops' : cpu_chunk,
'efficiency' : 1}])
samples.append(['sto', n, {'src' : None,
'rsize' : 0,
'tgt' : output_size,
'wsize' : bufsize,
'buf' : bufsize}])
info, ret, out = rs.emulate(samples=samples)
import os
import sys
import time
import pprint
import threading
import radical.synapse as rs
import radical.synapse.atoms as rsa
os.environ['RADICAL_SYNAPSE_DBURL'] = "file://%s" % os.getcwd()
print os.environ['RADICAL_SYNAPSE_DBURL']
print "------------------------------------------------------------------------"
os.environ['RADICAL_SYNAPSE_TAGS'] = "cpu"
info, _, _ = rs.emulate('test cpu')
pprint.pprint(info)
print "------------------------------------------------------------------------"
os.environ['RADICAL_SYNAPSE_TAGS'] = "mem"
info, _, _ = rs.emulate('test mem')
pprint.pprint(info)
print "------------------------------------------------------------------------"
os.environ['RADICAL_SYNAPSE_TAGS'] = "sto"
info, _, _ = rs.emulate('test sto')
pprint.pprint(info)
print "------------------------------------------------------------------------"
os.environ['RADICAL_SYNAPSE_TAGS'] = ""
info, _, _ = rs.emulate('test')
import os
import sys
import time
import pprint
import threading
import radical.synapse as rs
import radical.synapse.atoms as rsa
os.environ['RADICAL_SYNAPSE_DBURL'] = "file://%s" % os.getcwd ()
print os.environ['RADICAL_SYNAPSE_DBURL']
print "------------------------------------------------------------------------"
os.environ['RADICAL_SYNAPSE_TAGS'] = "cpu"
info, _, _ = rs.emulate ('test cpu')
pprint.pprint (info)
print "------------------------------------------------------------------------"
os.environ['RADICAL_SYNAPSE_TAGS'] = "mem"
info, _, _ = rs.emulate ('test mem')
pprint.pprint (info)
print "------------------------------------------------------------------------"
os.environ['RADICAL_SYNAPSE_TAGS'] = "sto"
info, _, _ = rs.emulate ('test sto')
pprint.pprint (info)
print "------------------------------------------------------------------------"
os.environ['RADICAL_SYNAPSE_TAGS'] = ""
info, _, _ = rs.emulate ('test')
import os
import sys
import time
import threading
from pprint import pprint
import radical.synapse as rs
import radical.synapse.atoms as rsa
os.environ['RADICAL_SYNAPSE_DBURL'] = "file://%s" % os.getcwd()
os.environ['RADICAL_SYNAPSE_TAGS'] = "cpu:cpu"
print os.environ['RADICAL_SYNAPSE_DBURL']
info, _, _ = rs.emulate('test cpu')
sys.exit
# ------------------------------------------------------------------------------
#
# info, ret, out = rs.profile ('md5sum /tmp/l')
info, _, _ = rs.emulate('md5sum /tmp/l')
pprint(info)
sys.exit()
# ------------------------------------------------------------------------------
#
def testme(delay):
def _testme(d):
import os
import sys
import time
import threading
from pprint import pprint
import radical.synapse as rs
import radical.synapse.atoms as rsa
os.environ['RADICAL_SYNAPSE_DBURL'] = "file://%s" % os.getcwd ()
os.environ['RADICAL_SYNAPSE_TAGS'] = "cpu:cpu"
print os.environ['RADICAL_SYNAPSE_DBURL']
info, _, _ = rs.emulate ('test cpu')
sys.exit
# ------------------------------------------------------------------------------
#
# info, ret, out = rs.profile ('md5sum /tmp/l')
info, _, _ = rs.emulate ('md5sum /tmp/l')
pprint (info)
sys.exit()
# ------------------------------------------------------------------------------
#
def testme (delay) :
