def ensure_files_in_store(dir_scan):
bundles = [
(str(dir_scan.path.remote_path), str(dir_scan.entry.cake),
dir_scan.bundle),
]
store_dir = True
while store_dir:
_, hashes_to_push = access.store_directories(directories=bundles)
store_dir = False
for h in hashes_to_push:
h = Cake.ensure_it(h)
name = dir_scan.bundle.get_name_by_cake(h)
file = os.path.join(dir_scan.path.fs_path, name)
fp = open(file, 'rb')
stored = access.write_content(fp)
if not stored.match(h):
log.info('path:%s, %s != %s' % (file, h, stored))
dir_scan.bundle[name] = Cake(stored.hash_bytes(),
h.header.type, h.header.role)
dir_scan.udk = dir_scan.bundle.cake()
store_dir = True
dir_scan.path.store_remote_path()
progress.just_processed(
sum(f.size for f in dir_scan.new_db_entries
if f.file_type == FileType.FILE) + dir_scan.bundle.size(),
dir_scan.path.fs_path)
def test_glue():
dbf = Dbf(GlueBase.metadata, test.file_path('test_glue.sqlite3'))
dbf.ensure_db()
glue_session = dbf.session()
shard_dbf = Dbf(CakeShardBase.metadata,
test.file_path('test_shard.sqlite3'))
shard_dbf.ensure_db()
shard_session = shard_dbf.session()
joe = User(id=Cake.new_portal(role=CakeRole.SYNAPSE),
email='*****@*****.**',
user_state=UserState.invitation,
passwd=SaltedSha.from_secret('xyz'))
cake = Cake.from_bytes(b'a' * 100)
portal = Portal(id=Cake.new_portal(), latest=cake)
shard_session.add(portal)
perm1 = Permission(
user=joe,
cake=portal.id,
permission_type=PermissionType.Read_,
)
perm2 = Permission(
user=joe,
cake=cake,
permission_type=PermissionType.Read_,
)
glue_session.add(perm1, perm2)
shard_session.commit()
glue_session.commit()
def create_portal(self, portal_id, cake):
'''
create portal pointing to cake, if portal is already exist -
fail.
This call could be used to edit portal if portal is owned
by user.
'''
portal_id = Cake.ensure_it(portal_id)
portal_id.assert_portal()
self.authorize(None, (PT.Create_Portals, ))
cake = Cake.ensure_it_or_none(cake)
cake_session = self.ctx.cake_session(portal_id)
portal_in_db = cake_session.query(Portal).\
filter(Portal.id == portal_id).one_or_none()
add_portal = portal_in_db is None
already_own = False
if not add_portal:
try:
self.authorize(portal_id, (PT.Own_Portal_, ))
already_own = True
except NotAuthorizedError:
pass
if add_portal or already_own:
dal.edit_portal(cake_session, Portal(id=portal_id, latest=cake),
self.auth_user)
if not already_own:
self.ctx.glue_session().add(
Permission(user=self.auth_user,
permission_type=PT.Own_Portal_,
cake=portal_id))
else:
raise AssertionError('portal %r is already exists.' % portal_id)
def store():
seed(0)
s = random_bytes(40)
eq_(len(s), 40)
w0 = hs.writer()
r0 = w0.write(s, done=True)
eq_(inline_udk, str(r0))
r0a = ContentAddress(Cake.from_bytes(s))
eq_(r0, r0a)
eq_(False, r0 == 0)
eq_(False, r0a == 0)
eq_(hash(r0), hash(r0a))
ok_(hs.lookup(Cake.from_bytes(s)).found())
w1 = hs.writer()
for _ in range(3):
w1.write(random_bytes(100))
r1 = w1.done()
s1 = str(r1)
eq_(db_udk, s1)
w2 = hs.writer()
for _ in range(100): # 100Mb
w2.write(random_bytes(1000))
w2.done()
r2 = w2.done() # call done twice
eq_(file_udk, str(r2))
return r0, r1, r2
def find_normal_user(glue_sess, user_or_email):
if isinstance(user_or_email, str) and '@' in user_or_email:
condition = User.email == user_or_email
else:
condition = User.id == Cake.ensure_it(user_or_email)
return glue_sess.query(User).filter(
condition, User.user_type == UserType.normal).one()
def login(self, email, passwd):
result = self.proxy.login(email=email,
passwd=passwd,
client_id=self.client_id)
self.email = email
self.session_id = Cake.ensure_it(result)
self.init_headers()
def grant_portal(self, portal_id, grantee, permission_type):
'''
grant Read_, Edit_Portal_ or Own_Portal_ permission to
grantee. `user has to have PT.Own_Portal_ or PT.Admin to do
this.
'''
portal_id = Cake.ensure_it(portal_id)
portal_id.assert_portal()
self.authorize(portal_id, (PT.Own_Portal_, PT.Admin))
portal = self.ctx.cake_session(portal_id).query(Portal).\
filter(Portal.id == portal_id).one()
if permission_type not in Permissions.portals:
raise AssertionError('pt:%r has to be one of %r' %
(permission_type, Permissions.portals))
grantee = self.ensure_user(grantee)
if portal is not None:
self.ctx.glue_session().add(
Permission(user=grantee,
permission_type=permission_type,
cake=portal.id))
else:
raise AssertionError('portal %r does not exists.' % portal_id)
def new_singleton(**ch_kwargs):
new_dmount = lambda: Cake.new_portal(**ch_kwargs)
class NewSingleton(NameIt, ReprIt):
single = Column(Integer, primary_key=True, default=1)
id = Column(StringCast(Cake), nullable=False, default=new_dmount)
return NewSingleton
def delete_portal(self, portal_id):
'''
disown portal, if nobody owns it deactivate it in
Portal table
'''
portal_id = Cake.ensure_it(portal_id)
portal_id.assert_portal()
self.authorize(portal_id, (PT.Own_Portal_, PT.Admin))
portal = self.ctx.cake_session(portal_id).query(Portal).\
filter(Portal.id == portal_id).one()
portal.active = False
def create_portal(self,
portal_id=None,
portal_role=None,
portal_type=None,
cake=None,
dir='.'):
self._check_cu_session(dir)
if portal_id is None:
portal_id = Cake.new_portal(role=portal_role, type=portal_type)
self.remote().create_portal(portal_id=portal_id, cake=cake)
print('Portal: {portal_id!s} \nCake: {cake!s}\n'.format(**locals()))
def __init__(self, s, _pt=None, _cake=None):
if s is None:
self.permission_type = _pt
self.cake = None
if self.permission_type.needs_cake():
self.cake = _cake
else:
p = s.split(':', 2)
self.permission_type = PermissionType[p[0]]
self.cake = Cake.ensure_it(p[1]) if len(p) == 2 else None
if self.permission_type.needs_cake() and self.cake is None:
raise ValueError('cake field is required for permission: %s' %
self.permission_type.name)
def __init__(self, path, from_db, stats):
Scan.__init__(self, path, FileType.FILE, stats)
stat = os.stat(self.path.fs_path)
self.entry.modtime = stat.st_mtime
self.entry.size = stat.st_size
if stats.force_rehash or from_db is None \
or self.entry.modtime > from_db.modtime:
self.stats.increment_count()
digest, inline_data = process_stream(
open(self.path.fs_path, 'rb'), on_chunk=self.stats.count_bytes)
self.entry.cake = Cake.from_digest_and_inline_data(
digest, inline_data)
else:
self.entry.cake = from_db.cake
def find(self, cake, dir='.'):
results = []
def find_recursively(directory, cake):
files = cscan.CakeEntries(directory).directory_usage()
for f in files:
path = os.path.join(directory, f.name)
if f.cake == cake:
e = {k: getattr(f, k) for k in 'file_type size'.split()}
e['path'] = path
results.append(e)
elif f.file_type == FileType.DIR:
find_recursively(path, cake)
find_recursively(dir, Cake.ensure_it(cake))
print_pad(results, 'file_type size path'.split())
def update_vtree(self, cake_path, cake=None, file=None, dir=None):
self._check_cu_session(dir or file or '.')
if cake is None:
digest, buff = process_stream(open(file, 'rb'))
cake = Cake.from_digest_and_inline_data(digest, buff)
unseen = self.remote().edit_portal_tree(files=[
(PatchAction.update, cake_path, cake),
])
for cake_to_write in map(Cake.ensure_it, unseen):
if cake != cake_to_write:
raise AssertionError(
'{cake} != {cake_to_write}'.format(**locals()))
elif file is None:
log.warning(' Server does not have %s stored.' % cake)
else:
fp = open(file, 'rb')
stored = self.remote().write_content(fp)
print('CPath: {cake_path!s} \nCake: {cake!s}\n'.format(**locals()))
def get_portal_tree(self, portal_id, asof_dt=None):
'''
read whole tree into `CakeTree`
'''
portal_id = Cake.ensure_it(portal_id)
if portal_id.type != CakeType.VTREE:
raise AssertionError("%s is not a TREE:%s " % (
portal_id,
portal_id.type,
))
VT = VolatileTree
if asof_dt is not None:
condition = and_(VT.portal_id == portal_id, VT.start_dt <= asof_dt,
or_(VT.end_dt == None, VT.end_dt > asof_dt))
else:
condition = and_(VT.portal_id == portal_id, VT.end_dt == None)
tree_paths = self.ctx.cake_session(portal_id).query(VT).filter(
condition).all()
tree = CakeTree(portal=portal_id)
for r in tree_paths:
tree[r.path] = r.cake
return tree
class PortalPk:
id = Column(StringCast(Cake),
primary_key=True,
default=lambda: Cake.new_portal(**ch_kwargs))
def __init__(self, parent, name, cake, state=NodeState.unknown):
Node.__init__(self, parent, name, state)
self._cake = Cake.ensure_it(cake)
class ServerKey(Singleton, ServerConfigBase):
secret = Column(StringCast(Cake), default=Cake.new_portal())
external_ip = Column(StringCast(InetAddress), nullable=True)
port = Column(Integer, nullable=False)
num_cake_shards = Column(Integer, nullable=False)