def compute(task_length): samples = list() samples.append( ['cpu', 0, { 'time': task_length, 'flops': 0, 'efficiency': 1 }]) info, ret, out = rs.emulate(samples=samples)
def read_compute_write(input_files, r_bufsize, task_length, output_files, output_sizes, w_bufsize): # we chunk the compute according to output chunking total_size = 0 for output_size in output_sizes: total_size += output_size nreads = int(total_size / bufsize) cpu_chunk = int(task_length / nreads) samples = list() n = 0 for input_file, output_file, output_size in zip(input_files, output_files, output_sizes): r_size = os.stat(input_file).st_size w_size = output_size n_samples = fsize / bufsize for s in range(n_samples): n += 2 samples.append([ 'sto', t, { 'src': input_file, 'rsize': int(r_size / n_samples), 'tgt': None, 'wsize': 0, 'buf': r_bufsize } ]) samples.append([ 'cpu', t + 1, { 'time': cpu_chunk, 'flops': 0, 'efficiency': 1 } ]) samples.append([ 'sto', t + 1, { 'src': None, 'rsize': 0, 'tgt': output_file, 'wsize': int(w_size / n_samples), 'buf': w_bufsize } ]) info, ret, out = rs.emulate(samples=samples)
def read_compute(input_files, bufsize, task_mode, task_length): total_size = 0 for input_file in input_files: total_size += os.stat(input_file).st_size nreads = int(total_size / bufsize) cpu_chunk = int(task_length / nreads) samples = list() n = 0 for input_file in input_files: fsize = os.stat(input_file).st_size / bufsize n_samples = fsize / bufsize for s in range(n_samples): # construct artificial time) n += 1 if task_mode == 'time': samples.append([ 'cpu', t, { 'time': cpu_chunk, 'flops': 0, 'efficiency': 1 } ]) elif task_mode == 'flops': samples.append([ 'cpu', t, { 'time': 0, 'flops': cpu_chunk, 'efficiency': 1 } ]) samples.append([ 'sto', t, { 'src': input_file, 'rsize': bufsize, 'tgt': None, 'wsize': 0, 'buf': bufsize } ]) info, ret, out = rs.emulate(samples=samples)
def writefiles(output_files, output_sizes, bufsize): samples = list() n = 0 for output_file, output_size in zip(output_files, output_sizes): n += 1 samples.append(['sto', n, {'src' : None, 'rsize' : 0, 'tgt' : output_file, 'wsize' : output_size, 'buf' : bufsize}]) info, ret, out = rs.emulate(samples=samples)
def compute_write(task_mode, task_length, output_files, output_sizes, bufsize): total_size = 0 for output_size in output_sizes: total_size += output_size nreads = int(total_size / bufsize) cpu_chunk = int(task_length / nreads) samples = list() n = 0 for output_file, output_size in zip(output_files, output_sizes): fsize = output_size n_samples = fsize / bufsize for s in range(n_samples): n += 1 if task_mode == 'time': samples.append([ 'cpu', n, { 'time': cpu_chunk, 'flops': 0, 'efficiency': 1 } ]) elif task_mode == 'flops': samples.append([ 'cpu', n, { 'time': 0, 'flops': cpu_chunk, 'efficiency': 1 } ]) samples.append([ 'sto', n, { 'src': None, 'rsize': 0, 'tgt': output_size, 'wsize': bufsize, 'buf': bufsize } ]) info, ret, out = rs.emulate(samples=samples)
def readfiles(input_files, bufsize): samples = list() n = 0 for input_file in input_files: n += 1 fsize = os.stat(input_file).st_size samples.append(['sto', n, {'src' : input_file, 'rsize' : fsize, 'tgt' : None, 'wsize' : 0, 'buf' : bufsize}]) info, ret, out = rs.emulate(samples=samples)
def compute(task_mode, task_length): samples = list() if task_mode == 'time' : samples.append(['cpu', 0, {'time' : task_length, 'flops' : 0, 'efficiency' : 1}]) elif task_mode == 'flops' : samples.append(['cpu', 0, {'time' : 0, 'flops' : task_length, 'efficiency' : 1}]) else: raise ValueError('Unknown task mode %s' % task_mode) info, ret, out = rs.emulate(samples=samples)
def read_compute_write(input_files, r_bufsize, task_mode, task_length, output_files, output_sizes, w_bufsize): # we chunk the compute according to output chunking total_size = 0 for output_size in output_sizes: total_size += output_size nreads = int(total_size / bufsize) cpu_chunk = int(task_length / nreads) samples = list() n = 0 for input_file, output_file, output_size in zip(input_files, output_files, output_sizes): r_size = os.stat(input_file).st_size w_size = output_size n_samples = fsize / bufsize for s in range(n_samples): n += 2 samples.append(['sto', t, {'src' : input_file, 'rsize' : int(r_size/n_samples), 'tgt' : None, 'wsize' : 0, 'buf' : r_bufsize}]) if task_mode == 'time': samples.append(['cpu', t+1, {'time' : cpu_chunk, 'flops' : 0, 'efficiency' : 1}]) elif task_mode == 'flops': samples.append(['cpu', t+1, {'time' : 0, 'flops' : cpu_chunk, 'efficiency' : 1}]) samples.append(['sto', t+1, {'src' : None, 'rsize' : 0, 'tgt' : output_file, 'wsize' : int(w_size/n_samples), 'buf' : w_bufsize}]) info, ret, out = rs.emulate(samples=samples)
def writefiles(output_files, output_sizes, bufsize): samples = list() n = 0 for output_file, output_size in zip(output_files, output_sizes): n += 1 samples.append([ 'sto', n, { 'src': None, 'rsize': 0, 'tgt': output_file, 'wsize': output_size, 'buf': bufsize } ]) info, ret, out = rs.emulate(samples=samples)
def readfiles(input_files, bufsize): samples = list() n = 0 for input_file in input_files: n += 1 fsize = os.stat(input_file).st_size samples.append([ 'sto', n, { 'src': input_file, 'rsize': fsize, 'tgt': None, 'wsize': 0, 'buf': bufsize } ]) info, ret, out = rs.emulate(samples=samples)
def compute(task_mode, task_length): samples = list() if task_mode == 'time': samples.append( ['cpu', 0, { 'time': task_length, 'flops': 0, 'efficiency': 1 }]) elif task_mode == 'flops': samples.append( ['cpu', 0, { 'time': 0, 'flops': task_length, 'efficiency': 1 }]) else: raise ValueError('Unknown task mode %s' % task_mode) info, ret, out = rs.emulate(samples=samples)
def read_compute(input_files, bufsize, task_mode, task_length): total_size = 0 for input_file in input_files: total_size += os.stat(input_file).st_size nreads = int(total_size / bufsize) cpu_chunk = int(task_length / nreads) samples = list() n = 0 for input_file in input_files: fsize = os.stat(input_file).st_size / bufsize n_samples = fsize / bufsize for s in range(n_samples): # construct artificial time) n += 1 if task_mode == 'time': samples.append(['cpu', t, {'time' : cpu_chunk, 'flops' : 0, 'efficiency' : 1}]) elif task_mode == 'flops': samples.append(['cpu', t, {'time' : 0, 'flops' : cpu_chunk, 'efficiency' : 1}]) samples.append(['sto', t, {'src' : input_file, 'rsize' : bufsize, 'tgt' : None, 'wsize' : 0, 'buf' : bufsize}]) info, ret, out = rs.emulate(samples=samples)
def compute_write(task_mode, task_length, output_files, output_sizes, bufsize): total_size = 0 for output_size in output_sizes: total_size += output_size nreads = int(total_size / bufsize) cpu_chunk = int(task_length / nreads) samples = list() n = 0 for output_file, output_size in zip(output_files, output_sizes): fsize = output_size n_samples = fsize / bufsize for s in range(n_samples): n += 1 if task_mode == 'time': samples.append(['cpu', n, {'time' : cpu_chunk, 'flops' : 0, 'efficiency' : 1}]) elif task_mode == 'flops': samples.append(['cpu', n, {'time' : 0, 'flops' : cpu_chunk, 'efficiency' : 1}]) samples.append(['sto', n, {'src' : None, 'rsize' : 0, 'tgt' : output_size, 'wsize' : bufsize, 'buf' : bufsize}]) info, ret, out = rs.emulate(samples=samples)
import os import sys import time import pprint import threading import radical.synapse as rs import radical.synapse.atoms as rsa os.environ['RADICAL_SYNAPSE_DBURL'] = "file://%s" % os.getcwd() print os.environ['RADICAL_SYNAPSE_DBURL'] print "------------------------------------------------------------------------" os.environ['RADICAL_SYNAPSE_TAGS'] = "cpu" info, _, _ = rs.emulate('test cpu') pprint.pprint(info) print "------------------------------------------------------------------------" os.environ['RADICAL_SYNAPSE_TAGS'] = "mem" info, _, _ = rs.emulate('test mem') pprint.pprint(info) print "------------------------------------------------------------------------" os.environ['RADICAL_SYNAPSE_TAGS'] = "sto" info, _, _ = rs.emulate('test sto') pprint.pprint(info) print "------------------------------------------------------------------------" os.environ['RADICAL_SYNAPSE_TAGS'] = "" info, _, _ = rs.emulate('test')
import os import sys import time import pprint import threading import radical.synapse as rs import radical.synapse.atoms as rsa os.environ['RADICAL_SYNAPSE_DBURL'] = "file://%s" % os.getcwd () print os.environ['RADICAL_SYNAPSE_DBURL'] print "------------------------------------------------------------------------" os.environ['RADICAL_SYNAPSE_TAGS'] = "cpu" info, _, _ = rs.emulate ('test cpu') pprint.pprint (info) print "------------------------------------------------------------------------" os.environ['RADICAL_SYNAPSE_TAGS'] = "mem" info, _, _ = rs.emulate ('test mem') pprint.pprint (info) print "------------------------------------------------------------------------" os.environ['RADICAL_SYNAPSE_TAGS'] = "sto" info, _, _ = rs.emulate ('test sto') pprint.pprint (info) print "------------------------------------------------------------------------" os.environ['RADICAL_SYNAPSE_TAGS'] = "" info, _, _ = rs.emulate ('test')
import os import sys import time import threading from pprint import pprint import radical.synapse as rs import radical.synapse.atoms as rsa os.environ['RADICAL_SYNAPSE_DBURL'] = "file://%s" % os.getcwd() os.environ['RADICAL_SYNAPSE_TAGS'] = "cpu:cpu" print os.environ['RADICAL_SYNAPSE_DBURL'] info, _, _ = rs.emulate('test cpu') sys.exit # ------------------------------------------------------------------------------ # # info, ret, out = rs.profile ('md5sum /tmp/l') info, _, _ = rs.emulate('md5sum /tmp/l') pprint(info) sys.exit() # ------------------------------------------------------------------------------ # def testme(delay): def _testme(d):
import os import sys import time import threading from pprint import pprint import radical.synapse as rs import radical.synapse.atoms as rsa os.environ['RADICAL_SYNAPSE_DBURL'] = "file://%s" % os.getcwd () os.environ['RADICAL_SYNAPSE_TAGS'] = "cpu:cpu" print os.environ['RADICAL_SYNAPSE_DBURL'] info, _, _ = rs.emulate ('test cpu') sys.exit # ------------------------------------------------------------------------------ # # info, ret, out = rs.profile ('md5sum /tmp/l') info, _, _ = rs.emulate ('md5sum /tmp/l') pprint (info) sys.exit() # ------------------------------------------------------------------------------ # def testme (delay) :