def build_dir_tree_chkeq(depth, startlevel): dirpaths = build_dir_tree_path(depth, startlevel) chkseq = pat_data_struct.get_empty_ChunkSeq() for dirpath in dirpaths: cbox = pat_data_struct.get_empty_ChunkBox2() op = {'opname': 'mkdir', 'optype': 'dir', 'opvalue': dirpath} # yes, I only put one operation to a chunkbox cbox['opseq'] = [op] chkseq['seq'].append(cbox) return chkseq
def build_dir_tree_chkeq( depth, startlevel ): dirpaths = build_dir_tree_path( depth, startlevel) chkseq = pat_data_struct.get_empty_ChunkSeq() for dirpath in dirpaths: cbox = pat_data_struct.get_empty_ChunkBox2() op = { 'opname':'mkdir', 'optype':'dir', 'opvalue':dirpath } # yes, I only put one operation to a chunkbox cbox['opseq'] = [op] chkseq['seq'].append(cbox) return chkseq
def build_file_chunkseq(file_treatment): """ ********************************************* PROVIDE ONLY THE MECHANISM, LEAST POLICY HERE TIRED IMPLEMENTING SIMILAR MECHANISM FOR DIFFERENT POLICIES ********************************************* file_treatment = { parent_dirid : fileid : make this globally unique writer_pid : writer pid (DEL)overlap : This one should be specified out of this function. It changes chunks chunks : [{'offset':, 'length':},{}] #chunk id is the index here write_order : [0,1,2,3,..] # The bitmaps apply to ordered chunkseq open_bitmap : [True, .. ] fsync_bitmap : [True, False, ...] close_bitmap : [True, .. ] sync_bitmap : [True, .. ] writer_cpu_map: [0,1,0,1] # set affinity to which cpu, # -1 means not schedule explicitly } This function returns a chunkseq of this treatment """ # logical space (setup chunkseq) nchunks = len(file_treatment['write_order']) chunkseq = pat_data_struct.get_empty_ChunkSeq() for pair in file_treatment['chunks']: cbox = pat_data_struct.get_empty_ChunkBox2() cbox['chunk']['offset'] = pair['offset'] cbox['chunk']['length'] = pair['length'] cbox['chunk']['fileid'] = file_treatment['fileid'] cbox['chunk']['parent_dirid'] = file_treatment['parent_dirid'] cbox['chunk']['filepath'] = os.path.join( #get_ladder_dir_path(file_treatment['parent_dirid']), get_dir_path(file_treatment['parent_dirid'], file_treatment['startlevel']), str(file_treatment['fileid']) + ".file") cbox['chunk']['writer_pid'] = file_treatment['writer_pid'] chunkseq['seq'].append(cbox) # Order it chunkseq['seq'] = [ chunkseq['seq'][i] \ for i in file_treatment['write_order'] ] # apply the bitmaps slotnames = ['A', '(', 'C', 'F', ')', 'S'] opbitmap = pat_data_struct.get_empty_OpBitmap() opbitmap['nchunks'] = nchunks for writer_cpu, open_bit, fsync_bit, close_bit, sync_bit\ in zip( file_treatment['writer_cpu_map'], file_treatment['open_bitmap'], file_treatment['fsync_bitmap'], file_treatment['close_bitmap'], file_treatment['sync_bitmap'] ): # each iteration in the loop is for a chunk d = { 'A': writer_cpu, '(': open_bit, 'C': 'C', 'F': fsync_bit, ')': close_bit, 'S': sync_bit } opbitmap['slotnames'].extend(slotnames) opbitmap['values'].extend([d[x] for x in slotnames]) #pprint.pprint(opbitmap) pattern_iter.assign_operations_to_chunkseq(chunkseq, opbitmap) return chunkseq
def build_file_chunkseq ( file_treatment ): """ ********************************************* PROVIDE ONLY THE MECHANISM, LEAST POLICY HERE TIRED IMPLEMENTING SIMILAR MECHANISM FOR DIFFERENT POLICIES ********************************************* file_treatment = { parent_dirid : fileid : make this globally unique writer_pid : writer pid (DEL)overlap : This one should be specified out of this function. It changes chunks chunks : [{'offset':, 'length':},{}] #chunk id is the index here write_order : [0,1,2,3,..] # The bitmaps apply to ordered chunkseq open_bitmap : [True, .. ] fsync_bitmap : [True, False, ...] close_bitmap : [True, .. ] sync_bitmap : [True, .. ] writer_cpu_map: [0,1,0,1] # set affinity to which cpu, # -1 means not schedule explicitly } This function returns a chunkseq of this treatment """ # logical space (setup chunkseq) nchunks = len(file_treatment['write_order']) chunkseq = pat_data_struct.get_empty_ChunkSeq() for pair in file_treatment['chunks']: cbox = pat_data_struct.get_empty_ChunkBox2() cbox['chunk']['offset'] = pair['offset'] cbox['chunk']['length'] = pair['length'] cbox['chunk']['fileid'] = file_treatment['fileid'] cbox['chunk']['parent_dirid'] = file_treatment['parent_dirid'] cbox['chunk']['filepath'] = os.path.join( #get_ladder_dir_path(file_treatment['parent_dirid']), get_dir_path(file_treatment['parent_dirid'], file_treatment['startlevel'] ), str( file_treatment['fileid'] ) + ".file" ) cbox['chunk']['writer_pid'] = file_treatment['writer_pid'] chunkseq['seq'].append( cbox ) # Order it chunkseq['seq'] = [ chunkseq['seq'][i] \ for i in file_treatment['write_order'] ] # apply the bitmaps slotnames = ['A', '(', 'C', 'F', ')', 'S'] opbitmap = pat_data_struct.get_empty_OpBitmap() opbitmap['nchunks'] = nchunks for writer_cpu, open_bit, fsync_bit, close_bit, sync_bit\ in zip( file_treatment['writer_cpu_map'], file_treatment['open_bitmap'], file_treatment['fsync_bitmap'], file_treatment['close_bitmap'], file_treatment['sync_bitmap'] ): # each iteration in the loop is for a chunk d = { 'A': writer_cpu, '(': open_bit, 'C': 'C', 'F': fsync_bit, ')': close_bit, 'S': sync_bit } opbitmap['slotnames'].extend( slotnames ) opbitmap['values'].extend( [ d[x] for x in slotnames ] ) #pprint.pprint(opbitmap) pattern_iter.assign_operations_to_chunkseq( chunkseq, opbitmap ) return chunkseq