def allocate(length, place, tm=None):
"""
1 最小代价找到一个可用文件, 从 _last_id 开始找起 -1 == stat('%d.lock')
2 try lock
3 stat again
# 应对一个目录下文件数目太多问题
- 在url中携带 bundle path 信息，并加密
data = open('sample.jpg', 'rb').read()
w = bundle.allocate(len(data), 'p')
w.ensure_url(prefix='fmn04', postfix='.jpg')
w.write(data)
del w # important to release the file lock
>>> if not os.path.exists('p'): os.makedirs('p')
>>> if os.path.exists('p/20110920/A'): os.unlink('p/20110920/A')
>>> data = open('sample.jpg', 'rb').read()
>>> w = allocate(len(data), place='p', tm=time.strptime('20110919', '%Y%m%d'))
>>> w.ensure_url(prefix='fmn04', postfix='.jpg')
>>> w.write(data)
83283
"""
global _last_id
if _last_id is None:
_last_id = os.getpid() % 10
if tm is None:
tm = time.localtime()
loop_count = 0
while True:
# 0
# 检测是否循环了一圈，采用备用方案: 直接分配一个超过 kBundleCountPerDay 的文件
loop_count += 1
if loop_count >= kBundleCountPerDay:
logging.info('allocation policy loop failed')
_last_id = kBundleCountPerDay + random.randint(1, 100)
# 1
bid = new_bid(_last_id % kBundleCountPerDay, tm)
fn_bundle = bundle_filename(bid, place)
try:
statinfo = os.stat(fn_bundle)
except OSError, e:
if e.errno != errno.ENOENT:
_last_id += 1 # TODO: lock
continue
statinfo = None
if statinfo and statinfo.st_size > kMaxBundleSize:
_last_id += 1 # TODO: lock
continue
# 2
fn_lock = '%s/%s' % (_lock_place, bid) # TODO: /place/.lock/sub-place/
dir_lock = os.path.dirname(fn_lock)
if not os.path.exists(dir_lock):
os.makedirs(dir_lock)
flock = FileLock(fn_lock, timeout=0.5)
try:
if not flock.try_acquire():
_last_id += 1 # TODO: lock
continue
except:
_last_id += 1 # TODO: lock
continue
assert flock.is_locked
# 3
if statinfo is None:
fp = create(fn_bundle)
fp.close()
offset = kBundleHeaderSize
else:
statinfo = os.stat(fn_bundle)
if statinfo.st_size > kMaxBundleSize:
_last_id += 1 # TODO: lock
continue
offset = statinfo.st_size
assert offset >= kBundleHeaderSize, offset
return Writer(flock, filename=fn_bundle, bid=bid, offset=offset, length=length)
def allocate(length, place, tm=None):
"""
1 最小代价找到一个可用文件, 从 _last_id 开始找起 -1 == stat('%d.lock')
2 try lock
3 stat again
# 应对一个目录下文件数目太多问题
- 在url中携带 bundle path 信息，并加密
data = open('sample.jpg', 'rb').read()
w = bundle.allocate(len(data), 'p')
w.ensure_url(prefix='fmn04', postfix='.jpg')
w.write(data)
del w # important to release the file lock
>>> if not os.path.exists('p'): os.makedirs('p')
>>> if os.path.exists('p/20110920/A'): os.unlink('p/20110920/A')
>>> data = open('sample.jpg', 'rb').read()
>>> w = allocate(len(data), place='p', tm=time.strptime('20110919', '%Y%m%d'))
>>> w.ensure_url(prefix='fmn04', postfix='.jpg')
>>> w.write(data)
83283
"""
global _last_id
if _last_id is None:
_last_id = os.getpid() % 10
if tm is None:
tm = time.localtime()
loop_count = 0
while True:
# 0
# 检测是否循环了一圈，采用备用方案: 直接分配一个超过 kBundleCountPerDay 的文件
loop_count += 1
if loop_count >= kBundleCountPerDay:
logging.info('allocation policy loop failed')
_last_id = kBundleCountPerDay + random.randint(1, 100)
# 1
bid = new_bid(_last_id % kBundleCountPerDay, tm)
fn_bundle = bundle_filename(bid, place)
try:
statinfo = os.stat(fn_bundle)
except OSError, e:
if e.errno != errno.ENOENT:
_last_id += 1 # TODO: lock
continue
statinfo = None
if statinfo and statinfo.st_size > kMaxBundleSize:
_last_id += 1 # TODO: lock
continue
# 2
fn_lock = '%s/%s' % (_lock_place, bid) # TODO: /place/.lock/sub-place/
dir_lock = os.path.dirname(fn_lock)
if not os.path.exists(dir_lock):
os.makedirs(dir_lock)
flock = FileLock(fn_lock, timeout=0.5)
try:
if not flock.try_acquire():
_last_id += 1 # TODO: lock
continue
except:
_last_id += 1 # TODO: lock
continue
assert flock.is_locked
# 3
if statinfo is None:
fp = create(fn_bundle)
fp.close()
offset = kBundleHeaderSize
else:
statinfo = os.stat(fn_bundle)
if statinfo.st_size > kMaxBundleSize:
_last_id += 1 # TODO: lock
continue
offset = statinfo.st_size
assert offset >= kBundleHeaderSize, offset
return Writer(flock,
filename=fn_bundle,
bid=bid,
offset=offset,
length=length)
