def build_dir_tree_chkeq(depth, startlevel):
dirpaths = build_dir_tree_path(depth, startlevel)
chkseq = pat_data_struct.get_empty_ChunkSeq()
for dirpath in dirpaths:
cbox = pat_data_struct.get_empty_ChunkBox2()
op = {'opname': 'mkdir', 'optype': 'dir', 'opvalue': dirpath}
# yes, I only put one operation to a chunkbox
cbox['opseq'] = [op]
chkseq['seq'].append(cbox)
return chkseq
def build_dir_tree_chkeq( depth, startlevel ):
dirpaths = build_dir_tree_path( depth, startlevel)
chkseq = pat_data_struct.get_empty_ChunkSeq()
for dirpath in dirpaths:
cbox = pat_data_struct.get_empty_ChunkBox2()
op = {
'opname':'mkdir',
'optype':'dir',
'opvalue':dirpath
}
# yes, I only put one operation to a chunkbox
cbox['opseq'] = [op]
chkseq['seq'].append(cbox)
return chkseq
def build_file_chunkseq(file_treatment):
"""
*********************************************
PROVIDE ONLY THE MECHANISM, LEAST POLICY HERE
TIRED IMPLEMENTING SIMILAR MECHANISM FOR DIFFERENT POLICIES
*********************************************
file_treatment = {
parent_dirid :
fileid : make this globally unique
writer_pid : writer pid
(DEL)overlap : This one should be specified out of this
function. It changes chunks
chunks : [{'offset':, 'length':},{}]
#chunk id is the index here
write_order : [0,1,2,3,..]
# The bitmaps apply to ordered chunkseq
open_bitmap : [True, .. ]
fsync_bitmap : [True, False, ...]
close_bitmap : [True, .. ]
sync_bitmap : [True, .. ]
writer_cpu_map: [0,1,0,1] # set affinity to which cpu,
# -1 means not schedule explicitly
}
This function returns a chunkseq of this treatment
"""
# logical space (setup chunkseq)
nchunks = len(file_treatment['write_order'])
chunkseq = pat_data_struct.get_empty_ChunkSeq()
for pair in file_treatment['chunks']:
cbox = pat_data_struct.get_empty_ChunkBox2()
cbox['chunk']['offset'] = pair['offset']
cbox['chunk']['length'] = pair['length']
cbox['chunk']['fileid'] = file_treatment['fileid']
cbox['chunk']['parent_dirid'] = file_treatment['parent_dirid']
cbox['chunk']['filepath'] = os.path.join(
#get_ladder_dir_path(file_treatment['parent_dirid']),
get_dir_path(file_treatment['parent_dirid'],
file_treatment['startlevel']),
str(file_treatment['fileid']) + ".file")
cbox['chunk']['writer_pid'] = file_treatment['writer_pid']
chunkseq['seq'].append(cbox)
# Order it
chunkseq['seq'] = [ chunkseq['seq'][i] \
for i in file_treatment['write_order'] ]
# apply the bitmaps
slotnames = ['A', '(', 'C', 'F', ')', 'S']
opbitmap = pat_data_struct.get_empty_OpBitmap()
opbitmap['nchunks'] = nchunks
for writer_cpu, open_bit, fsync_bit, close_bit, sync_bit\
in zip(
file_treatment['writer_cpu_map'],
file_treatment['open_bitmap'],
file_treatment['fsync_bitmap'],
file_treatment['close_bitmap'],
file_treatment['sync_bitmap'] ):
# each iteration in the loop is for a chunk
d = {
'A': writer_cpu,
'(': open_bit,
'C': 'C',
'F': fsync_bit,
')': close_bit,
'S': sync_bit
}
opbitmap['slotnames'].extend(slotnames)
opbitmap['values'].extend([d[x] for x in slotnames])
#pprint.pprint(opbitmap)
pattern_iter.assign_operations_to_chunkseq(chunkseq, opbitmap)
return chunkseq
def build_file_chunkseq ( file_treatment ):
"""
*********************************************
PROVIDE ONLY THE MECHANISM, LEAST POLICY HERE
TIRED IMPLEMENTING SIMILAR MECHANISM FOR DIFFERENT POLICIES
*********************************************
file_treatment = {
parent_dirid :
fileid : make this globally unique
writer_pid : writer pid
(DEL)overlap : This one should be specified out of this
function. It changes chunks
chunks : [{'offset':, 'length':},{}]
#chunk id is the index here
write_order : [0,1,2,3,..]
# The bitmaps apply to ordered chunkseq
open_bitmap : [True, .. ]
fsync_bitmap : [True, False, ...]
close_bitmap : [True, .. ]
sync_bitmap : [True, .. ]
writer_cpu_map: [0,1,0,1] # set affinity to which cpu,
# -1 means not schedule explicitly
}
This function returns a chunkseq of this treatment
"""
# logical space (setup chunkseq)
nchunks = len(file_treatment['write_order'])
chunkseq = pat_data_struct.get_empty_ChunkSeq()
for pair in file_treatment['chunks']:
cbox = pat_data_struct.get_empty_ChunkBox2()
cbox['chunk']['offset'] = pair['offset']
cbox['chunk']['length'] = pair['length']
cbox['chunk']['fileid'] = file_treatment['fileid']
cbox['chunk']['parent_dirid'] = file_treatment['parent_dirid']
cbox['chunk']['filepath'] = os.path.join(
#get_ladder_dir_path(file_treatment['parent_dirid']),
get_dir_path(file_treatment['parent_dirid'],
file_treatment['startlevel']
),
str( file_treatment['fileid'] ) + ".file" )
cbox['chunk']['writer_pid'] = file_treatment['writer_pid']
chunkseq['seq'].append( cbox )
# Order it
chunkseq['seq'] = [ chunkseq['seq'][i] \
for i in file_treatment['write_order'] ]
# apply the bitmaps
slotnames = ['A', '(', 'C', 'F', ')', 'S']
opbitmap = pat_data_struct.get_empty_OpBitmap()
opbitmap['nchunks'] = nchunks
for writer_cpu, open_bit, fsync_bit, close_bit, sync_bit\
in zip(
file_treatment['writer_cpu_map'],
file_treatment['open_bitmap'],
file_treatment['fsync_bitmap'],
file_treatment['close_bitmap'],
file_treatment['sync_bitmap'] ):
# each iteration in the loop is for a chunk
d = {
'A': writer_cpu,
'(': open_bit,
'C': 'C',
'F': fsync_bit,
')': close_bit,
'S': sync_bit
}
opbitmap['slotnames'].extend( slotnames )
opbitmap['values'].extend( [ d[x] for x in slotnames ] )
#pprint.pprint(opbitmap)
pattern_iter.assign_operations_to_chunkseq( chunkseq, opbitmap )
return chunkseq
