def build_dir_tree_chkeq(depth, startlevel):
dirpaths = build_dir_tree_path(depth, startlevel)
chkseq = pat_data_struct.get_empty_ChunkSeq()
for dirpath in dirpaths:
cbox = pat_data_struct.get_empty_ChunkBox2()
op = {'opname': 'mkdir', 'optype': 'dir', 'opvalue': dirpath}
# yes, I only put one operation to a chunkbox
cbox['opseq'] = [op]
chkseq['seq'].append(cbox)
return chkseq
def build_dir_tree_chkeq( depth, startlevel ):
dirpaths = build_dir_tree_path( depth, startlevel)
chkseq = pat_data_struct.get_empty_ChunkSeq()
for dirpath in dirpaths:
cbox = pat_data_struct.get_empty_ChunkBox2()
op = {
'opname':'mkdir',
'optype':'dir',
'opvalue':dirpath
}
# yes, I only put one operation to a chunkbox
cbox['opseq'] = [op]
chkseq['seq'].append(cbox)
return chkseq
def build_file_chunkseq(file_treatment):
"""
*********************************************
PROVIDE ONLY THE MECHANISM, LEAST POLICY HERE
TIRED IMPLEMENTING SIMILAR MECHANISM FOR DIFFERENT POLICIES
*********************************************
file_treatment = {
parent_dirid :
fileid : make this globally unique
writer_pid : writer pid
(DEL)overlap : This one should be specified out of this
function. It changes chunks
chunks : [{'offset':, 'length':},{}]
#chunk id is the index here
write_order : [0,1,2,3,..]
# The bitmaps apply to ordered chunkseq
open_bitmap : [True, .. ]
fsync_bitmap : [True, False, ...]
close_bitmap : [True, .. ]
sync_bitmap : [True, .. ]
writer_cpu_map: [0,1,0,1] # set affinity to which cpu,
# -1 means not schedule explicitly
}
This function returns a chunkseq of this treatment
"""
# logical space (setup chunkseq)
nchunks = len(file_treatment['write_order'])
chunkseq = pat_data_struct.get_empty_ChunkSeq()
for pair in file_treatment['chunks']:
cbox = pat_data_struct.get_empty_ChunkBox2()
cbox['chunk']['offset'] = pair['offset']
cbox['chunk']['length'] = pair['length']
cbox['chunk']['fileid'] = file_treatment['fileid']
cbox['chunk']['parent_dirid'] = file_treatment['parent_dirid']
cbox['chunk']['filepath'] = os.path.join(
#get_ladder_dir_path(file_treatment['parent_dirid']),
get_dir_path(file_treatment['parent_dirid'],
file_treatment['startlevel']),
str(file_treatment['fileid']) + ".file")
cbox['chunk']['writer_pid'] = file_treatment['writer_pid']
chunkseq['seq'].append(cbox)
# Order it
chunkseq['seq'] = [ chunkseq['seq'][i] \
for i in file_treatment['write_order'] ]
# apply the bitmaps
slotnames = ['A', '(', 'C', 'F', ')', 'S']
opbitmap = pat_data_struct.get_empty_OpBitmap()
opbitmap['nchunks'] = nchunks
for writer_cpu, open_bit, fsync_bit, close_bit, sync_bit\
in zip(
file_treatment['writer_cpu_map'],
file_treatment['open_bitmap'],
file_treatment['fsync_bitmap'],
file_treatment['close_bitmap'],
file_treatment['sync_bitmap'] ):
# each iteration in the loop is for a chunk
d = {
'A': writer_cpu,
'(': open_bit,
'C': 'C',
'F': fsync_bit,
')': close_bit,
'S': sync_bit
}
opbitmap['slotnames'].extend(slotnames)
opbitmap['values'].extend([d[x] for x in slotnames])
#pprint.pprint(opbitmap)
pattern_iter.assign_operations_to_chunkseq(chunkseq, opbitmap)
return chunkseq
def build_conf(treatment, confparser):
"""
This function build a confparser from the treatment.
The treatment contains values for different factors.
Refer to ** Design the experiment for the paper ** in
Evernote for more details.
Note that this function only takes one treatment (a
point on the region). The distribution of the treatments
are controlled out of this function.
Implementing factors in treatment:
1. n_dir_depth: we name the directory tree by the index
of directory in the pre-order traversal. No two dir names
are the same.
treatment = {
filesystem:
disksize :
free_space_layout_score:
free_space_ratio:
n_dir_depth:
# file id in file_treatment is the index here
files: [file_treatment0, file_treatment1, ..],
# the number of item in the following list
# is the number of total chunks of all files
filechunk_order: [0, 2, 1, fileid,..]
}
Here I utilize the convenient structure of confparser to store treatment
info.
"""
if not confparser.has_section('system'):
confparser.add_section('system')
if not confparser.has_section('workload'):
confparser.add_section('workload')
confparser.set('system', 'filesystem', treatment['filesystem'])
confparser.set('system', 'disksize', str(treatment['disksize']))
confparser.set('system', 'disk_used', str(treatment['disk_used']))
confparser.set('system', 'makeloopdevice', 'yes')
confparser.set('system', 'layoutnumber', str(treatment['layoutnumber']))
confparser.set('system', 'mountopts', treatment['mountopts'])
confparser.set('system', 'core.count', str(treatment['core.count']))
chkseq = pat_data_struct.get_empty_ChunkSeq()
# creat directory tree
dirs_chkseq = build_dir_tree_chkeq(treatment['dir_depth'],
treatment['startlevel'])
chkseq['seq'].extend(dirs_chkseq['seq'])
# Get chunkseq for each file
nfiles = len(treatment['files'])
files_chkseq_list = []
for file_treatment in treatment['files']:
file_treatment['startlevel'] = treatment['startlevel']
files_chkseq_list.append(build_file_chunkseq(file_treatment))
# mix the chunks of all files
ckpos = [0] * nfiles
for curfile in treatment['filechunk_order']:
# curfile is not file id.
# it is the position in files_chkseq_list
chkseq['seq'].append(files_chkseq_list[curfile]['seq'][ckpos[curfile]])
ckpos[curfile] += 1
confparser.set('workload', 'files_chkseq', str(chkseq))
def build_file_chunkseq ( file_treatment ):
"""
*********************************************
PROVIDE ONLY THE MECHANISM, LEAST POLICY HERE
TIRED IMPLEMENTING SIMILAR MECHANISM FOR DIFFERENT POLICIES
*********************************************
file_treatment = {
parent_dirid :
fileid : make this globally unique
writer_pid : writer pid
(DEL)overlap : This one should be specified out of this
function. It changes chunks
chunks : [{'offset':, 'length':},{}]
#chunk id is the index here
write_order : [0,1,2,3,..]
# The bitmaps apply to ordered chunkseq
open_bitmap : [True, .. ]
fsync_bitmap : [True, False, ...]
close_bitmap : [True, .. ]
sync_bitmap : [True, .. ]
writer_cpu_map: [0,1,0,1] # set affinity to which cpu,
# -1 means not schedule explicitly
}
This function returns a chunkseq of this treatment
"""
# logical space (setup chunkseq)
nchunks = len(file_treatment['write_order'])
chunkseq = pat_data_struct.get_empty_ChunkSeq()
for pair in file_treatment['chunks']:
cbox = pat_data_struct.get_empty_ChunkBox2()
cbox['chunk']['offset'] = pair['offset']
cbox['chunk']['length'] = pair['length']
cbox['chunk']['fileid'] = file_treatment['fileid']
cbox['chunk']['parent_dirid'] = file_treatment['parent_dirid']
cbox['chunk']['filepath'] = os.path.join(
#get_ladder_dir_path(file_treatment['parent_dirid']),
get_dir_path(file_treatment['parent_dirid'],
file_treatment['startlevel']
),
str( file_treatment['fileid'] ) + ".file" )
cbox['chunk']['writer_pid'] = file_treatment['writer_pid']
chunkseq['seq'].append( cbox )
# Order it
chunkseq['seq'] = [ chunkseq['seq'][i] \
for i in file_treatment['write_order'] ]
# apply the bitmaps
slotnames = ['A', '(', 'C', 'F', ')', 'S']
opbitmap = pat_data_struct.get_empty_OpBitmap()
opbitmap['nchunks'] = nchunks
for writer_cpu, open_bit, fsync_bit, close_bit, sync_bit\
in zip(
file_treatment['writer_cpu_map'],
file_treatment['open_bitmap'],
file_treatment['fsync_bitmap'],
file_treatment['close_bitmap'],
file_treatment['sync_bitmap'] ):
# each iteration in the loop is for a chunk
d = {
'A': writer_cpu,
'(': open_bit,
'C': 'C',
'F': fsync_bit,
')': close_bit,
'S': sync_bit
}
opbitmap['slotnames'].extend( slotnames )
opbitmap['values'].extend( [ d[x] for x in slotnames ] )
#pprint.pprint(opbitmap)
pattern_iter.assign_operations_to_chunkseq( chunkseq, opbitmap )
return chunkseq
def build_conf ( treatment, confparser ):
"""
This function build a confparser from the treatment.
The treatment contains values for different factors.
Refer to ** Design the experiment for the paper ** in
Evernote for more details.
Note that this function only takes one treatment (a
point on the region). The distribution of the treatments
are controlled out of this function.
Implementing factors in treatment:
1. n_dir_depth: we name the directory tree by the index
of directory in the pre-order traversal. No two dir names
are the same.
treatment = {
filesystem:
disksize :
free_space_layout_score:
free_space_ratio:
n_dir_depth:
# file id in file_treatment is the index here
files: [file_treatment0, file_treatment1, ..],
# the number of item in the following list
# is the number of total chunks of all files
filechunk_order: [0, 2, 1, fileid,..]
}
Here I utilize the convenient structure of confparser to store treatment
info.
"""
if not confparser.has_section('system'):
confparser.add_section('system')
if not confparser.has_section('workload'):
confparser.add_section('workload')
confparser.set('system', 'filesystem', treatment['filesystem'])
confparser.set('system', 'disksize' , str(treatment['disksize']))
confparser.set('system', 'disk_used',
str(treatment['disk_used']))
confparser.set('system', 'makeloopdevice', 'yes')
confparser.set('system', 'layoutnumber',
str(treatment['layoutnumber']))
confparser.set('system', 'mountopts', treatment['mountopts'])
confparser.set('system', 'core.count', str(treatment['core.count']))
chkseq = pat_data_struct.get_empty_ChunkSeq()
# creat directory tree
dirs_chkseq = build_dir_tree_chkeq( treatment['dir_depth'],
treatment['startlevel'] )
chkseq['seq'].extend( dirs_chkseq['seq'] )
# Get chunkseq for each file
nfiles = len( treatment['files'] )
files_chkseq_list = []
for file_treatment in treatment['files']:
file_treatment['startlevel'] = treatment['startlevel']
files_chkseq_list.append(
build_file_chunkseq( file_treatment ) )
# mix the chunks of all files
ckpos = [0] * nfiles
for curfile in treatment['filechunk_order']:
# curfile is not file id.
# it is the position in files_chkseq_list
chkseq['seq'].append(
files_chkseq_list[curfile]['seq'][ ckpos[curfile] ])
ckpos[curfile] += 1
confparser.set('workload', 'files_chkseq', str(chkseq))
