def get(self): while True: try: line = self.cur.get() except NotDataException: self.close() self.cur.unlink() self.create_cur() self.cur = AccountNotMemery(self.CUR_NAME) continue if not line: continue return line
def init(self): self.init_path() self.data_fds = [] if not osp.exists(self.data_path): os.makedirs(self.data_path) for i in range(self.DATA_NUM): p = osp.join(self.data_path, str(i)) self.data_fds.append(file(p, "a")) self.exists_fd = file(self.exists_path, "a") self.exists_ct = 0 self.create_cur() self.cur = AccountNotMemery(self.CUR_NAME) self.cur.trunk()
class Store(object): DATA_NUM = 100 DATA_NAME = "data" CUR_NAME = "cur" EXISTS_NAME = "exists%s.txt" def __init__(self, data_path="."): if data_path: self.path = data_path else: self.path = osp.dirname(osp.dirname(__file__)) self.path = osp.join(self.path, "filesytem") self.distinct_base = DistinctBase(osp.join(self.path, "base")) self.save_ct = 0 self.lock = threading.RLock() self.init() def init_path(self): self.data_path = osp.join(self.path, self.DATA_NAME) self.exists_path = osp.join(self.path, self.EXISTS_NAME % datetime.today().strftime("%y%m%d")) AccountNotMemery.BASEDIR = self.path self.cur_file = osp.join(self.path, "%s.txt" % self.CUR_NAME) def get_file_size(self, filename): with file(filename) as f: f.seek(0, 2) return f.tell() def init(self): self.init_path() self.data_fds = [] if not osp.exists(self.data_path): os.makedirs(self.data_path) for i in range(self.DATA_NUM): p = osp.join(self.data_path, str(i)) self.data_fds.append(file(p, "a")) self.exists_fd = file(self.exists_path, "a") self.exists_ct = 0 self.create_cur() self.cur = AccountNotMemery(self.CUR_NAME) self.cur.trunk() def create_cur(self): logging.info("begin create cur") filenames = [f.name for f in self.data_fds] output = file(self.cur_file, "a") num = self.distinct_base(filenames, lambda x: int(x[-2:], 16), output) output.close() for i, f in enumerate(filenames): self.data_fds[i] = file(f, "w") def close(self): for f in self.data_fds: f.close() self.exists_fd.flush() self.sync() def load(self, limit=10000): pass def export(self, filename): pass def sync(self): self.cur.sync() self.exists_fd.flush() def get(self): while True: try: line = self.cur.get() except NotDataException: self.close() self.cur.unlink() self.create_cur() self.cur = AccountNotMemery(self.CUR_NAME) continue if not line: continue return line def __len__(self): return 99999 def save(self, user, names=None): if names: for name in names: try: index = int(name[-2:], 16) % self.DATA_NUM except: traceback.print_exc() continue self.data_fds[index].write(name) self.data_fds[index].write("\n") self.save_ct += 1 if user is not None: if not names: return self.exists_fd.write(user) self.exists_fd.write("\n") self.exists_ct += 1 def __del__(self): self.close()
def trunk(name): tos = AccountNotMemery(name) tos.load() tos.trunk()