def consume (self, e_iter, c_iter, chunk_size, db_meta_data) :
assert sos.path.exists (self.x_uri.name), self.x_uri.name
assert not self.info.commits
assert not self.info.pending
assert not self.info.stores
db_uri = self.db_uri
x_name = self.x_uri.name
with TFL.lock_file (x_name) :
info = self.info
stores = info.stores = []
commits = info.commits = []
for i, cargo in enumerate (sliced (e_iter, chunk_size)) :
s_name = TFL.Filename ("by_pid_%d" % i, self.x_uri)
with open (s_name.name, "wb") as file :
pickle.dump (cargo, file, pyk.pickle_protocol)
stores.append (s_name.base_ext)
for cargo in sliced (c_iter, chunk_size) :
max_cid = cargo [-1] [1] ["cid"]
c_name = TFL.Filename ("%d.commit" % max_cid, self.x_uri)
with open (c_name.name, "wb") as file :
pickle.dump (cargo, file, pyk.pickle_protocol)
commits.append ((max_cid, c_name.base_ext))
info.max_cid = db_meta_data.max_cid
info.max_pid = db_meta_data.max_pid
self._save_info (info)
def consume (self, e_iter, c_iter, chunk_size, db_meta_data) :
assert sos.path.exists (self.x_uri), self.x_uri.name
assert not self.info.commits
assert not self.info.pending
assert not self.info.stores
db_uri = self.db_uri
x_uri = self.x_uri
with TFL.lock_file (x_uri) :
info = self.info
stores = info.stores = []
commits = info.commits = []
for i, cargo in enumerate (sliced (e_iter, chunk_size)) :
s_name = TFL.Filename ("by_pid_%d" % i, x_uri)
with open (s_name, "wb") as file :
pickle.dump (cargo, file, pyk.pickle_protocol)
stores.append (s_name.base_ext)
for cargo in sliced (c_iter, chunk_size) :
max_cid = cargo [-1] [1] ["cid"]
c_name = TFL.Filename ("%d.commit" % max_cid, x_uri)
with open (c_name, "wb") as file :
pickle.dump (cargo, file, pyk.pickle_protocol)
commits.append ((max_cid, c_name.base_ext))
info.max_cid = db_meta_data.max_cid
info.max_pid = db_meta_data.max_pid
self._save_info (info)
def create (self) :
assert not sos.path.exists (self.db_uri), self.db_uri.name
assert not sos.path.exists (self.x_uri), self.x_uri.name
x_uri = self.x_uri
with TFL.lock_file (x_uri) :
sos.mkdir (x_uri)
sos.system ("chmod go= %s" % x_uri.name)
self._create_info ()
def create (self) :
assert not sos.path.exists (self.db_uri.name), self.db_uri.name
assert not sos.path.exists (self.x_uri.name), self.x_uri.name
x_name = self.x_uri.name
with TFL.lock_file (x_name) :
sos.mkdir (x_name)
sos.system ("chmod go= %s" % x_name)
self._create_info ()
def produce_entities (self) :
assert sos.path.exists (self.x_uri), self.x_uri.name
assert not self.info.pending
info = self.info
x_uri = self.x_uri
with TFL.lock_file (x_uri) :
for s in info.stores :
with open (TFL.Filename (s, x_uri), "rb") as file :
yield from pickle.load (file)
def load_changes (self) :
assert sos.path.exists (self.x_uri.name), self.x_uri.name
info = self.info
x_uri = self.x_uri
with TFL.lock_file (x_uri.name) :
for (cid, name) in info.commits :
file_name = TFL.Filename (name, x_uri).name
for c in self._loaded_changes (file_name) :
pass ### `_loaded_changes` adds the changes to `self.cm`
self.cm.to_load = []
def produce_entities (self) :
assert sos.path.exists (self.x_uri.name), self.x_uri.name
assert not self.info.pending
info = self.info
x_uri = self.x_uri
with TFL.lock_file (x_uri.name) :
for s in info.stores :
with open (TFL.Filename (s, x_uri).name, "rb") as file :
for epc in pickle.load (file) :
yield epc
def _save_context (self, x_name, scope, info) :
Version = self.Version
with TFL.lock_file (x_name) :
self._check_sync_ro (info)
yield info
ems = scope.ems
info.max_cid = ems.max_cid
info.max_pid = ems.max_pid
info.max_surrs = ems.max_surrs
self._save_info (info)
def _save_context (self, x_uri, scope, info) :
Version = self.Version
with TFL.lock_file (x_uri) :
self._check_sync_ro (info)
yield info
ems = scope.ems
info.max_cid = ems.max_cid
info.max_pid = ems.max_pid
info.max_surrs = ems.max_surrs
self._save_info (info)
def load_changes (self) :
assert sos.path.exists (self.x_uri), self.x_uri.name
info = self.info
x_uri = self.x_uri
with TFL.lock_file (x_uri) :
for (cid, name) in info.commits :
file_name = TFL.Filename (name, x_uri)
for c in self._loaded_changes (file_name) :
pass ### `_loaded_changes` adds the changes to `self.cm`
self.cm.to_load = []
def produce_changes (self) :
assert sos.path.exists (self.x_uri), self.x_uri.name
assert not self.info.pending
info = self.info
x_uri = self.x_uri
with TFL.lock_file (x_uri) :
for (cid, name) in info.commits + info.pending :
file_name = TFL.Filename (name, x_uri)
with open (file_name, "rb") as file :
changes = pickle.load (file)
yield from changes
def load_info (self) :
assert sos.path.exists (self.db_uri.name), self.db_uri.name
x_name = self.x_uri.name
with TFL.lock_file (x_name) :
if not sos.path.exists (x_name) :
sos.mkdir (x_name)
sos.system ("chmod go= %s" % x_name)
with contextlib.closing \
(self.ZF.ZipFile (self.db_uri.name, "r")) as zf :
zf.extractall (x_name)
self.info = self._load_info ()
def load_info (self) :
assert sos.path.exists (self.db_uri), self.db_uri.name
x_uri = self.x_uri
with TFL.lock_file (x_uri) :
if not sos.path.exists (x_uri) :
sos.mkdir (x_uri)
sos.system ("chmod go= %s" % x_uri.name)
with contextlib.closing \
(self.ZF.ZipFile (self.db_uri, "r")) as zf :
zf.extractall (x_uri)
self.info = self._load_info ()
def load_objects (self) :
assert sos.path.exists (self.x_uri), self.x_uri.name
info = self.info
x_uri = self.x_uri
with TFL.lock_file (x_uri) :
self.scope.db_errors = []
for s in info.stores :
self._load_store (TFL.Filename (s, x_uri))
for (cid, name) in info.pending :
self._load_pending (TFL.Filename (name, x_uri))
self.cm.to_load = [name for (cid, name) in info.commits]
def load_objects (self) :
assert sos.path.exists (self.x_uri.name), self.x_uri.name
info = self.info
x_uri = self.x_uri
with TFL.lock_file (x_uri.name) :
self.scope.db_errors = []
for s in info.stores :
self._load_store (TFL.Filename (s, x_uri).name)
for (cid, name) in info.pending :
self._load_pending (TFL.Filename (name, x_uri).name)
self.cm.to_load = [name for (cid, name) in info.commits]
def produce_changes (self) :
assert sos.path.exists (self.x_uri.name), self.x_uri.name
assert not self.info.pending
info = self.info
x_uri = self.x_uri
with TFL.lock_file (x_uri.name) :
for (cid, name) in info.commits + info.pending :
file_name = TFL.Filename (name, x_uri).name
with open (file_name, "rb") as file :
changes = pickle.load (file)
for cargo in changes :
yield cargo
def close (self) :
assert sos.path.exists (self.x_uri), self.x_uri.name
db_uri = self.db_uri
x_uri = self.x_uri
bak = TFL.Filename (".bak", db_uri)
with TFL.lock_file (x_uri) :
info = self.info
self._check_sync (info)
with TFL.open_to_replace \
(db_uri, mode = "wb", backup_name = bak) as file:
sos.fchmod (file.fileno (), stat.S_IRUSR | stat.S_IWUSR)
with contextlib.closing (self.ZF.ZipFile (file, "w")) as zf :
for abs, rel in info.FILES (self.x_uri, self.info_uri) :
zf.write (abs, rel)
sos.rmdir (x_uri, deletefiles = True)
def close (self) :
assert sos.path.exists (self.x_uri.name), self.x_uri.name
db_uri = self.db_uri
x_name = self.x_uri.name
bak = TFL.Filename (".bak", db_uri).name
with TFL.lock_file (x_name) :
info = self.info
self._check_sync (info)
with TFL.open_to_replace \
(db_uri.name, mode = "wb", backup_name = bak) as file:
sos.fchmod (file.fileno (), stat.S_IRUSR | stat.S_IWUSR)
with contextlib.closing (self.ZF.ZipFile (file, "w")) as zf :
for abs, rel in info.FILES (self.x_uri, self.info_uri) :
zf.write (abs, rel)
sos.rmdir (x_name, deletefiles = True)
def change_readonly (self, state) :
with TFL.lock_file (self.x_uri) :
info = self.info
self._check_sync (info)
info.readonly = bool (state)
self._save_info (info)
def change_readonly (self, state) :
with TFL.lock_file (self.x_uri.name) :
info = self.info
self._check_sync (info)
info.readonly = bool (state)
self._save_info (info)
