def build_dir_tree_chkeq(depth, startlevel): dirpaths = build_dir_tree_path(depth, startlevel) chkseq = pat_data_struct.get_empty_ChunkSeq() for dirpath in dirpaths: cbox = pat_data_struct.get_empty_ChunkBox2() op = {'opname': 'mkdir', 'optype': 'dir', 'opvalue': dirpath} # yes, I only put one operation to a chunkbox cbox['opseq'] = [op] chkseq['seq'].append(cbox) return chkseq
def build_dir_tree_chkeq( depth, startlevel ): dirpaths = build_dir_tree_path( depth, startlevel) chkseq = pat_data_struct.get_empty_ChunkSeq() for dirpath in dirpaths: cbox = pat_data_struct.get_empty_ChunkBox2() op = { 'opname':'mkdir', 'optype':'dir', 'opvalue':dirpath } # yes, I only put one operation to a chunkbox cbox['opseq'] = [op] chkseq['seq'].append(cbox) return chkseq
def build_file_chunkseq(file_treatment): """ ********************************************* PROVIDE ONLY THE MECHANISM, LEAST POLICY HERE TIRED IMPLEMENTING SIMILAR MECHANISM FOR DIFFERENT POLICIES ********************************************* file_treatment = { parent_dirid : fileid : make this globally unique writer_pid : writer pid (DEL)overlap : This one should be specified out of this function. It changes chunks chunks : [{'offset':, 'length':},{}] #chunk id is the index here write_order : [0,1,2,3,..] # The bitmaps apply to ordered chunkseq open_bitmap : [True, .. ] fsync_bitmap : [True, False, ...] close_bitmap : [True, .. ] sync_bitmap : [True, .. ] writer_cpu_map: [0,1,0,1] # set affinity to which cpu, # -1 means not schedule explicitly } This function returns a chunkseq of this treatment """ # logical space (setup chunkseq) nchunks = len(file_treatment['write_order']) chunkseq = pat_data_struct.get_empty_ChunkSeq() for pair in file_treatment['chunks']: cbox = pat_data_struct.get_empty_ChunkBox2() cbox['chunk']['offset'] = pair['offset'] cbox['chunk']['length'] = pair['length'] cbox['chunk']['fileid'] = file_treatment['fileid'] cbox['chunk']['parent_dirid'] = file_treatment['parent_dirid'] cbox['chunk']['filepath'] = os.path.join( #get_ladder_dir_path(file_treatment['parent_dirid']), get_dir_path(file_treatment['parent_dirid'], file_treatment['startlevel']), str(file_treatment['fileid']) + ".file") cbox['chunk']['writer_pid'] = file_treatment['writer_pid'] chunkseq['seq'].append(cbox) # Order it chunkseq['seq'] = [ chunkseq['seq'][i] \ for i in file_treatment['write_order'] ] # apply the bitmaps slotnames = ['A', '(', 'C', 'F', ')', 'S'] opbitmap = pat_data_struct.get_empty_OpBitmap() opbitmap['nchunks'] = nchunks for writer_cpu, open_bit, fsync_bit, close_bit, sync_bit\ in zip( file_treatment['writer_cpu_map'], file_treatment['open_bitmap'], file_treatment['fsync_bitmap'], file_treatment['close_bitmap'], file_treatment['sync_bitmap'] ): # each iteration in the loop is for a chunk d = { 'A': writer_cpu, '(': open_bit, 'C': 'C', 'F': fsync_bit, ')': close_bit, 'S': sync_bit } opbitmap['slotnames'].extend(slotnames) opbitmap['values'].extend([d[x] for x in slotnames]) #pprint.pprint(opbitmap) pattern_iter.assign_operations_to_chunkseq(chunkseq, opbitmap) return chunkseq
def build_conf(treatment, confparser): """ This function build a confparser from the treatment. The treatment contains values for different factors. Refer to ** Design the experiment for the paper ** in Evernote for more details. Note that this function only takes one treatment (a point on the region). The distribution of the treatments are controlled out of this function. Implementing factors in treatment: 1. n_dir_depth: we name the directory tree by the index of directory in the pre-order traversal. No two dir names are the same. treatment = { filesystem: disksize : free_space_layout_score: free_space_ratio: n_dir_depth: # file id in file_treatment is the index here files: [file_treatment0, file_treatment1, ..], # the number of item in the following list # is the number of total chunks of all files filechunk_order: [0, 2, 1, fileid,..] } Here I utilize the convenient structure of confparser to store treatment info. """ if not confparser.has_section('system'): confparser.add_section('system') if not confparser.has_section('workload'): confparser.add_section('workload') confparser.set('system', 'filesystem', treatment['filesystem']) confparser.set('system', 'disksize', str(treatment['disksize'])) confparser.set('system', 'disk_used', str(treatment['disk_used'])) confparser.set('system', 'makeloopdevice', 'yes') confparser.set('system', 'layoutnumber', str(treatment['layoutnumber'])) confparser.set('system', 'mountopts', treatment['mountopts']) confparser.set('system', 'core.count', str(treatment['core.count'])) chkseq = pat_data_struct.get_empty_ChunkSeq() # creat directory tree dirs_chkseq = build_dir_tree_chkeq(treatment['dir_depth'], treatment['startlevel']) chkseq['seq'].extend(dirs_chkseq['seq']) # Get chunkseq for each file nfiles = len(treatment['files']) files_chkseq_list = [] for file_treatment in treatment['files']: file_treatment['startlevel'] = treatment['startlevel'] files_chkseq_list.append(build_file_chunkseq(file_treatment)) # mix the chunks of all files ckpos = [0] * nfiles for curfile in treatment['filechunk_order']: # curfile is not file id. # it is the position in files_chkseq_list chkseq['seq'].append(files_chkseq_list[curfile]['seq'][ckpos[curfile]]) ckpos[curfile] += 1 confparser.set('workload', 'files_chkseq', str(chkseq))
def build_file_chunkseq ( file_treatment ): """ ********************************************* PROVIDE ONLY THE MECHANISM, LEAST POLICY HERE TIRED IMPLEMENTING SIMILAR MECHANISM FOR DIFFERENT POLICIES ********************************************* file_treatment = { parent_dirid : fileid : make this globally unique writer_pid : writer pid (DEL)overlap : This one should be specified out of this function. It changes chunks chunks : [{'offset':, 'length':},{}] #chunk id is the index here write_order : [0,1,2,3,..] # The bitmaps apply to ordered chunkseq open_bitmap : [True, .. ] fsync_bitmap : [True, False, ...] close_bitmap : [True, .. ] sync_bitmap : [True, .. ] writer_cpu_map: [0,1,0,1] # set affinity to which cpu, # -1 means not schedule explicitly } This function returns a chunkseq of this treatment """ # logical space (setup chunkseq) nchunks = len(file_treatment['write_order']) chunkseq = pat_data_struct.get_empty_ChunkSeq() for pair in file_treatment['chunks']: cbox = pat_data_struct.get_empty_ChunkBox2() cbox['chunk']['offset'] = pair['offset'] cbox['chunk']['length'] = pair['length'] cbox['chunk']['fileid'] = file_treatment['fileid'] cbox['chunk']['parent_dirid'] = file_treatment['parent_dirid'] cbox['chunk']['filepath'] = os.path.join( #get_ladder_dir_path(file_treatment['parent_dirid']), get_dir_path(file_treatment['parent_dirid'], file_treatment['startlevel'] ), str( file_treatment['fileid'] ) + ".file" ) cbox['chunk']['writer_pid'] = file_treatment['writer_pid'] chunkseq['seq'].append( cbox ) # Order it chunkseq['seq'] = [ chunkseq['seq'][i] \ for i in file_treatment['write_order'] ] # apply the bitmaps slotnames = ['A', '(', 'C', 'F', ')', 'S'] opbitmap = pat_data_struct.get_empty_OpBitmap() opbitmap['nchunks'] = nchunks for writer_cpu, open_bit, fsync_bit, close_bit, sync_bit\ in zip( file_treatment['writer_cpu_map'], file_treatment['open_bitmap'], file_treatment['fsync_bitmap'], file_treatment['close_bitmap'], file_treatment['sync_bitmap'] ): # each iteration in the loop is for a chunk d = { 'A': writer_cpu, '(': open_bit, 'C': 'C', 'F': fsync_bit, ')': close_bit, 'S': sync_bit } opbitmap['slotnames'].extend( slotnames ) opbitmap['values'].extend( [ d[x] for x in slotnames ] ) #pprint.pprint(opbitmap) pattern_iter.assign_operations_to_chunkseq( chunkseq, opbitmap ) return chunkseq
def build_conf ( treatment, confparser ): """ This function build a confparser from the treatment. The treatment contains values for different factors. Refer to ** Design the experiment for the paper ** in Evernote for more details. Note that this function only takes one treatment (a point on the region). The distribution of the treatments are controlled out of this function. Implementing factors in treatment: 1. n_dir_depth: we name the directory tree by the index of directory in the pre-order traversal. No two dir names are the same. treatment = { filesystem: disksize : free_space_layout_score: free_space_ratio: n_dir_depth: # file id in file_treatment is the index here files: [file_treatment0, file_treatment1, ..], # the number of item in the following list # is the number of total chunks of all files filechunk_order: [0, 2, 1, fileid,..] } Here I utilize the convenient structure of confparser to store treatment info. """ if not confparser.has_section('system'): confparser.add_section('system') if not confparser.has_section('workload'): confparser.add_section('workload') confparser.set('system', 'filesystem', treatment['filesystem']) confparser.set('system', 'disksize' , str(treatment['disksize'])) confparser.set('system', 'disk_used', str(treatment['disk_used'])) confparser.set('system', 'makeloopdevice', 'yes') confparser.set('system', 'layoutnumber', str(treatment['layoutnumber'])) confparser.set('system', 'mountopts', treatment['mountopts']) confparser.set('system', 'core.count', str(treatment['core.count'])) chkseq = pat_data_struct.get_empty_ChunkSeq() # creat directory tree dirs_chkseq = build_dir_tree_chkeq( treatment['dir_depth'], treatment['startlevel'] ) chkseq['seq'].extend( dirs_chkseq['seq'] ) # Get chunkseq for each file nfiles = len( treatment['files'] ) files_chkseq_list = [] for file_treatment in treatment['files']: file_treatment['startlevel'] = treatment['startlevel'] files_chkseq_list.append( build_file_chunkseq( file_treatment ) ) # mix the chunks of all files ckpos = [0] * nfiles for curfile in treatment['filechunk_order']: # curfile is not file id. # it is the position in files_chkseq_list chkseq['seq'].append( files_chkseq_list[curfile]['seq'][ ckpos[curfile] ]) ckpos[curfile] += 1 confparser.set('workload', 'files_chkseq', str(chkseq))