def allocate(length, place, tm=None): """ 1 最小代价找到一个可用文件, 从 _last_id 开始找起 -1 == stat('%d.lock') 2 try lock 3 stat again # 应对一个目录下文件数目太多问题 - 在url中携带 bundle path 信息,并加密 data = open('sample.jpg', 'rb').read() w = bundle.allocate(len(data), 'p') w.ensure_url(prefix='fmn04', postfix='.jpg') w.write(data) del w # important to release the file lock >>> if not os.path.exists('p'): os.makedirs('p') >>> if os.path.exists('p/20110920/A'): os.unlink('p/20110920/A') >>> data = open('sample.jpg', 'rb').read() >>> w = allocate(len(data), place='p', tm=time.strptime('20110919', '%Y%m%d')) >>> w.ensure_url(prefix='fmn04', postfix='.jpg') >>> w.write(data) 83283 """ global _last_id if _last_id is None: _last_id = os.getpid() % 10 if tm is None: tm = time.localtime() loop_count = 0 while True: # 0 # 检测是否循环了一圈,采用备用方案: 直接分配一个超过 kBundleCountPerDay 的文件 loop_count += 1 if loop_count >= kBundleCountPerDay: logging.info('allocation policy loop failed') _last_id = kBundleCountPerDay + random.randint(1, 100) # 1 bid = new_bid(_last_id % kBundleCountPerDay, tm) fn_bundle = bundle_filename(bid, place) try: statinfo = os.stat(fn_bundle) except OSError, e: if e.errno != errno.ENOENT: _last_id += 1 # TODO: lock continue statinfo = None if statinfo and statinfo.st_size > kMaxBundleSize: _last_id += 1 # TODO: lock continue # 2 fn_lock = '%s/%s' % (_lock_place, bid) # TODO: /place/.lock/sub-place/ dir_lock = os.path.dirname(fn_lock) if not os.path.exists(dir_lock): os.makedirs(dir_lock) flock = FileLock(fn_lock, timeout=0.5) try: if not flock.try_acquire(): _last_id += 1 # TODO: lock continue except: _last_id += 1 # TODO: lock continue assert flock.is_locked # 3 if statinfo is None: fp = create(fn_bundle) fp.close() offset = kBundleHeaderSize else: statinfo = os.stat(fn_bundle) if statinfo.st_size > kMaxBundleSize: _last_id += 1 # TODO: lock continue offset = statinfo.st_size assert offset >= kBundleHeaderSize, offset return Writer(flock, filename=fn_bundle, bid=bid, offset=offset, length=length)