def __init__(self, dbdir):
"""
Init function
"""
super(Database, self).__init__()
# internal db dir, it contains the real
self.directory = dbdir
self.medialist = MediaList()
# create or open db
self._db = db.Database(self.directory + '/db')
def __init__(self, dbdir):
"""
Init function
"""
super(Database, self).__init__()
# internal db dir, it contains the real
self.directory = dbdir
self.medialist = MediaList()
# create or open db
self._db = db.Database(self.directory + '/db')
def __init__(self, dbdir):
"""
Init function
"""
super(Database, self).__init__()
# internal db dir, it contains the real db and the
# overlay dir for the beacon
self.directory = dbdir
self.medialist = MediaList()
# create or open db
overlay = os.path.join(self.directory, 'overlays')
if not os.path.isdir(overlay):
os.makedirs(overlay)
self._db = db.Database(self.directory + '/db')
class Database(kaa.Object):
"""
Database API for the client side, providing read-only access to the
beacon database.
This class is subclassed by the server for the read/write database.
"""
def __init__(self, dbdir):
"""
Init function
"""
super(Database, self).__init__()
# internal db dir, it contains the real
self.directory = dbdir
self.medialist = MediaList()
# create or open db
self._db = db.Database(self.directory + '/db')
def commit():
"""
Stub on the client side: implemented in the server db
"""
pass
def add_object(*args, **kwargs):
"""
Stub on the client side: implemented in the server db
"""
pass
def delete_object(*args, **kwargs):
"""
Stub on the client side: implemented in the server db
"""
pass
def acquire_read_lock(self):
"""
Stub on the client side: implemented in the server db
"""
return kaa.InProgress().finish(None)
def md5url(self, url, subdir):
"""
Convert url into md5 sum
"""
if url.startswith('http://'):
subdir += '/%s/' % url[7:url[7:].find('/')+7]
fname = hashlib.md5(url).hexdigest() + os.path.splitext(url)[1]
return os.path.join(self.directory, subdir, fname)
def get_db_info(self):
"""
Returns information about the database. Look at
kaa.db.Database.get_db_info() for more details.
"""
info = self._db.get_db_info()
info['directory'] = self.directory
return info
# -------------------------------------------------------------------------
# Query functions
#
# The query functions can modify the database when in server mode. E.g.
# a directory query could detect deleted files and will delete them in
# the database. In client mode, the query functions will use the database
# read only.
# -------------------------------------------------------------------------
def query(self, **query):
"""
Main query function. This function will call one of the specific
query functions in this class depending on the query. This function
returns an InProgress.
"""
# Remove non-true recursive attribute from query (non-recursive is default).
if not query.get('recursive', True):
del query['recursive']
# Passed by caller to collect list of deleted items for directory query.
garbage = query.pop('garbage', None)
# do query based on type
if query.keys() == ['filename']:
fname = os.path.realpath(query['filename'])
return kaa.InProgress().execute(self.query_filename, fname)
if query.keys() == ['id']:
return kaa.InProgress().execute(self._db_query_id, query['id'])
if sorted(query.keys()) == ['parent', 'recursive']:
if not query['parent']._beacon_isdir:
raise AttributeError('parent is no directory')
return self._db_query_dir_recursive(query['parent'], garbage)
if 'parent' in query:
if len(query) == 1:
if query['parent']._beacon_isdir:
return self._db_query_dir(query['parent'], garbage)
query['parent'] = query['parent']._beacon_id
if 'media' not in query and query.get('type') != 'media':
# query only media we have right now
query['media'] = db.QExpr('in', self.medialist.get_all_beacon_ids())
elif query.get('media') == 'all':
del query['media']
if 'attr' in query:
return kaa.InProgress().execute(self._db_query_attr, query)
if query.get('type') == 'media':
return kaa.InProgress().execute(self._db.query, **query)
return self._db_query_raw(query)
def query_media(self, media):
"""
Get media information.
"""
if hasattr(media, 'id'):
# object is a media object
id = media.id
else:
# object is only an id
id = media
media = None
result = self._db.query(type="media", name=id)
if not result:
return None
result = result[0]
if not media:
return result
# TODO: it's a bit ugly to set url here, but we have no other choice
media.url = result['content'] + '://' + media.mountpoint
dbid = ('media', result['id'])
media._beacon_id = dbid
root = self._db.query(parent=dbid)[0]
if root['type'] == 'dir':
media.root = create_directory(root, media)
else:
media.root = create_item(root, media)
return result
@kaa.coroutine()
def _db_query_dir(self, parent, garbage):
"""
A query to get all files in a directory. The parameter parent is a
directort object.
"""
if parent._beacon_islink:
# WARNING: parent is a link, we need to follow it
dirname = os.path.realpath(parent.filename)
parent = self.query_filename(dirname)
if not parent._beacon_isdir:
# oops, this is not directory anymore, return nothing
yield []
else:
dirname = parent.filename[:-1]
listing = parent._beacon_listdir()
items = []
if parent._beacon_id:
items = [ create_by_type(i, parent, i['type'] == 'dir') \
for i in self._db.query(parent = parent._beacon_id) ]
# sort items based on name. The listdir is also sorted by name,
# that makes checking much faster
items.sort(lambda x,y: cmp(x._beacon_name, y._beacon_name))
# TODO: use parent mtime to check if an update is needed. Maybe call
# it scan time or something like that. Also make it an option so the
# user can turn the feature off.
yield self.acquire_read_lock()
pos = -1
for f, fullname, stat_res in listing[0]:
pos += 1
isdir = stat.S_ISDIR(stat_res[stat.ST_MODE])
if pos == len(items):
# new file at the end
if isdir:
items.append(create_directory(f, parent))
continue
items.append(create_file(f, parent))
continue
while pos < len(items) and f > items[pos]._beacon_name:
# file deleted
i = items[pos]
if not i.isdir and not i.isfile:
# A remote URL in the directory
pos += 1
continue
items.remove(i)
# Server only: delete from database by adding it to the
# internal changes list. It will be deleted right before the
# next commit.
self.delete_object(i)
if garbage is not None:
garbage.append(i)
if pos < len(items) and f == items[pos]._beacon_name:
# same file
continue
# new file
if isdir:
items.insert(pos, create_directory(f, parent))
continue
items.insert(pos, create_file(f, parent))
if pos + 1 < len(items):
# deleted files at the end
for i in items[pos+1-len(items):]:
if not i.isdir and not i.isfile:
# A remote URL in the directory
continue
items.remove(i)
# Server only: delete from database by adding it to the
# internal changes list. It will be deleted right before the
# next commit.
self.delete_object(i)
if garbage is not None:
garbage.append(i)
# no need to sort the items again, they are already sorted based
# on name, let us keep it that way. And name is unique in a directory.
# items.sort(lambda x,y: cmp(x.url, y.url))
yield items
@kaa.coroutine()
def _db_query_dir_recursive(self, parent, garbage):
"""
Return all files in the directory 'parent' including files in
subdirectories (and so on). The directories itself will not be
returned. If a subdir is a softlink, it will be skipped. This
query does not check if the files are still there and if the
database list is up to date.
"""
if parent._beacon_islink:
# WARNING: parent is a link, we need to follow it
dirname = os.path.realpath(parent.filename)
parent = self.query_filename(dirname)
if not parent._beacon_isdir:
# oops, this is not directory anymore, return nothing
yield []
else:
dirname = parent.filename[:-1]
timer = time.time()
items = []
# A list of all directories we will look at. If a link is in the
# directory it will be ignored.
directories = [ parent ]
while directories:
parent = directories.pop(0)
for i in (yield self._db_query_dir(parent, garbage)):
if i.isdir and not i._beacon_islink:
directories.append(child)
items.append(i)
if time.time() > timer + 0.1:
# we used too much time. Call yield NotFinished at
# this point to continue later.
timer = time.time()
yield kaa.NotFinished
# sort items based on name. The listdir is also sorted by name,
# that makes checking much faster
items.sort(lambda x,y: cmp(x._beacon_name, y._beacon_name))
yield items
def _db_query_id(self, (type, id), cache=None):
class Database(kaa.Object):
"""
Database API for the client side, providing read-only access to the
beacon database.
This class is subclassed by the server for the read/write database.
"""
def __init__(self, dbdir):
"""
Init function
"""
super(Database, self).__init__()
# internal db dir, it contains the real
self.directory = dbdir
self.medialist = MediaList()
# create or open db
self._db = db.Database(self.directory + '/db')
def commit():
"""
Stub on the client side: implemented in the server db
"""
pass
def add_object(*args, **kwargs):
"""
Stub on the client side: implemented in the server db
"""
pass
def delete_object(*args, **kwargs):
"""
Stub on the client side: implemented in the server db
"""
pass
def acquire_read_lock(self):
"""
Stub on the client side: implemented in the server db
"""
return kaa.InProgress().finish(None)
def md5url(self, url, subdir):
"""
Convert url into md5 sum
"""
if url.startswith('http://'):
subdir += '/%s/' % url[7:url[7:].find('/')+7]
fname = hashlib.md5(url).hexdigest() + os.path.splitext(url)[1]
return os.path.join(self.directory, subdir, fname)
def get_db_info(self):
"""
Returns information about the database. Look at
kaa.db.Database.get_db_info() for more details.
"""
info = self._db.get_db_info()
info['directory'] = self.directory
return info
# -------------------------------------------------------------------------
# Query functions
#
# The query functions can modify the database when in server mode. E.g.
# a directory query could detect deleted files and will delete them in
# the database. In client mode, the query functions will use the database
# read only.
# -------------------------------------------------------------------------
def query(self, **query):
"""
Main query function. This function will call one of the specific
query functions in this class depending on the query. This function
returns an InProgress.
"""
# Remove non-true recursive attribute from query (non-recursive is default).
if not query.get('recursive', True):
del query['recursive']
# Passed by caller to collect list of deleted items for directory query.
garbage = query.pop('garbage', None)
# do query based on type
if query.keys() == ['filename']:
fname = os.path.realpath(query['filename'])
return kaa.InProgress().execute(self.query_filename, fname)
if query.keys() == ['id']:
return kaa.InProgress().execute(self._db_query_id, query['id'])
if sorted(query.keys()) == ['parent', 'recursive']:
if not query['parent']._beacon_isdir:
raise AttributeError('parent is no directory')
return self._db_query_dir_recursive(query['parent'], garbage)
if 'parent' in query:
if len(query) == 1:
if query['parent']._beacon_isdir:
return self._db_query_dir(query['parent'], garbage)
query['parent'] = query['parent']._beacon_id
if 'media' not in query and query.get('type') != 'media':
# query only media we have right now
query['media'] = db.QExpr('in', self.medialist.get_all_beacon_ids())
elif query.get('media') == 'all':
del query['media']
if 'attr' in query:
return kaa.InProgress().execute(self._db_query_attr, query)
if query.get('type') == 'media':
return kaa.InProgress().execute(self._db.query, **query)
return self._db_query_raw(query)
def query_media(self, media):
"""
Get media information.
"""
if hasattr(media, 'id'):
# object is a media object
id = media.id
else:
# object is only an id
id = media
media = None
result = self._db.query(type="media", name=id)
if not result:
return None
result = result[0]
if not media:
return result
# TODO: it's a bit ugly to set url here, but we have no other choice
media.url = result['content'] + '://' + media.mountpoint
dbid = ('media', result['id'])
media._beacon_id = dbid
root = self._db.query(parent=dbid)[0]
if root['type'] == 'dir':
media.root = create_directory(root, media)
else:
media.root = create_item(root, media)
return result
@kaa.coroutine()
def _db_query_dir(self, parent, garbage):
"""
A query to get all files in a directory. The parameter parent is a
directort object.
"""
if parent._beacon_islink:
# WARNING: parent is a link, we need to follow it
dirname = os.path.realpath(parent.filename)
parent = self.query_filename(dirname)
if not parent._beacon_isdir:
# oops, this is not directory anymore, return nothing
yield []
else:
dirname = parent.filename[:-1]
listing = parent._beacon_listdir()
items = []
if parent._beacon_id:
items = [ create_by_type(i, parent, i['type'] == 'dir') \
for i in self._db.query(parent = parent._beacon_id) ]
# sort items based on name. The listdir is also sorted by name,
# that makes checking much faster
items.sort(lambda x,y: cmp(x._beacon_name, y._beacon_name))
# TODO: use parent mtime to check if an update is needed. Maybe call
# it scan time or something like that. Also make it an option so the
# user can turn the feature off.
yield self.acquire_read_lock()
pos = -1
for f, fullname, stat_res in listing[0]:
pos += 1
isdir = stat.S_ISDIR(stat_res[stat.ST_MODE])
if pos == len(items):
# new file at the end
if isdir:
items.append(create_directory(f, parent))
continue
items.append(create_file(f, parent))
continue
while pos < len(items) and f > items[pos]._beacon_name:
# file deleted
i = items[pos]
if not i.isdir and not i.isfile:
# A remote URL in the directory
pos += 1
continue
items.remove(i)
# Server only: delete from database by adding it to the
# internal changes list. It will be deleted right before the
# next commit.
self.delete_object(i)
if garbage is not None:
garbage.append(i)
if pos < len(items) and f == items[pos]._beacon_name:
# same file
continue
# new file
if isdir:
items.insert(pos, create_directory(f, parent))
continue
items.insert(pos, create_file(f, parent))
if pos + 1 < len(items):
# deleted files at the end
for i in items[pos+1-len(items):]:
if not i.isdir and not i.isfile:
# A remote URL in the directory
continue
items.remove(i)
# Server only: delete from database by adding it to the
# internal changes list. It will be deleted right before the
# next commit.
self.delete_object(i)
if garbage is not None:
garbage.append(i)
# no need to sort the items again, they are already sorted based
# on name, let us keep it that way. And name is unique in a directory.
# items.sort(lambda x,y: cmp(x.url, y.url))
yield items
@kaa.coroutine()
def _db_query_dir_recursive(self, parent, garbage):
"""
Return all files in the directory 'parent' including files in
subdirectories (and so on). The directories itself will not be
returned. If a subdir is a softlink, it will be skipped. This
query does not check if the files are still there and if the
database list is up to date.
"""
if parent._beacon_islink:
# WARNING: parent is a link, we need to follow it
dirname = os.path.realpath(parent.filename)
parent = self.query_filename(dirname)
if not parent._beacon_isdir:
# oops, this is not directory anymore, return nothing
yield []
else:
dirname = parent.filename[:-1]
timer = time.time()
items = []
# A list of all directories we will look at. If a link is in the
# directory it will be ignored.
directories = [ parent ]
while directories:
parent = directories.pop(0)
for i in (yield self._db_query_dir(parent, garbage)):
if i.isdir and not i._beacon_islink:
directories.append(child)
items.append(i)
if time.time() > timer + 0.1:
# we used too much time. Call yield NotFinished at
# this point to continue later.
timer = time.time()
yield kaa.NotFinished
# sort items based on name. The listdir is also sorted by name,
# that makes checking much faster
items.sort(lambda x,y: cmp(x._beacon_name, y._beacon_name))
yield items
def _db_query_id(self, (type, id), cache=None):