def get_dataset_files(d,conn=sddb.conn,limit=None):
"""
Retrieves all dataset's files
Args
limit: if set, returns only a subset of datasets's files
"""
files=[]
c = conn.cursor()
limit_clause="limit %i"%limit if limit is not None else ""
q="select * from file where dataset_id = %i order by variable %s" % (d.dataset_id,limit_clause)
c.execute(q)
rs=c.fetchone()
while rs!=None:
files.append(sdsqlutils.get_object_from_resultset(rs,File))
rs=c.fetchone()
c.close()
return files
def get_files(limit=None,
conn=sddb.conn,
**search_constraints): # don't change arguments order here
"""
Notes
- one search constraint must be given at least
- if 'limit' is None, retrieve all records matching the search constraints
"""
files = []
search_placeholder = sdsqlutils.build_search_placeholder(
search_constraints)
orderby = "priority DESC, checksum"
limit_clause = "limit %i" % limit if limit is not None else ""
c = conn.cursor()
q = "select * from file where %s order by %s %s" % (search_placeholder,
orderby, limit_clause)
c.execute(q, search_constraints)
rs = c.fetchone()
while rs != None:
files.append(sdsqlutils.get_object_from_resultset(rs, File))
rs = c.fetchone()
c.close()
return files
def get_datasets(limit=None,
conn=sddb.conn,
**search_constraints): # don't change arguments order here
"""
Note
If 'limit' is None, retrieve all records matching the search constraints
"""
datasets = []
c = conn.cursor()
limit_clause = "limit %i" % limit if limit is not None else ""
if len(search_constraints) > 0:
q = "select * from dataset where %s order by path asc %s" % (
sdsqlutils.build_search_placeholder(search_constraints),
limit_clause)
c.execute(q, search_constraints)
else:
q = "select * from dataset order by path asc %s" % limit_clause
c.execute(q)
rs = c.fetchone()
while rs != None:
datasets.append(sdsqlutils.get_object_from_resultset(rs, Dataset))
rs = c.fetchone()
c.close()
return datasets
def get_files_pagination(conn=sddb.conn):
"""
this method is used to loop over all files (note that we use pagination here not to load all the rows in memory !!!)
note
this method is like get_files_batch(), but use pagination instead of using yield
"""
global pagination_offset
files=[]
c = conn.cursor()
q="select * from file order by file_id ASC limit %d offset %d" % (pagination_limit,pagination_offset)
# debug
#print q
c.execute(q)
results = c.fetchall()
for rs in results:
files.append(sdsqlutils.get_object_from_resultset(rs,File))
c.close()
# move OFFSET for the next call
pagination_offset+=pagination_block_size
return files
def get_dataset_files(d,conn=sddb.conn,limit=None):
"""
Retrieves all dataset's files
Args
limit: if set, returns only a subset of datasets's files
"""
files=[]
c = conn.cursor()
limit_clause="limit %i"%limit if limit is not None else ""
q="select * from file where dataset_id = %i order by variable %s" % (d.dataset_id,limit_clause)
c.execute(q)
rs=c.fetchone()
while rs!=None:
files.append(sdsqlutils.get_object_from_resultset(rs,File))
rs=c.fetchone()
c.close()
return files
def get_datasets(limit=None,conn=sddb.conn,**search_constraints): # don't change arguments order here
"""
Note
If 'limit' is None, retrieve all records matching the search constraints
"""
datasets=[]
c = conn.cursor()
limit_clause="limit %i"%limit if limit is not None else ""
if len(search_constraints)>0:
q="select * from dataset where %s order by path asc %s"%(sdsqlutils.build_search_placeholder(search_constraints),limit_clause)
c.execute(q,search_constraints)
else:
q="select * from dataset order by path asc %s"%limit_clause
c.execute(q)
rs=c.fetchone()
while rs!=None:
datasets.append(sdsqlutils.get_object_from_resultset(rs,Dataset))
rs=c.fetchone()
c.close()
return datasets
def get_files(self):
files = []
results = self.get_items()
for rs in results:
f = sdsqlutils.get_object_from_resultset(rs, sdtypes.File)
files.append(f)
return files
def get_dataset(path=None,dataset_id=None,dataset_functional_id=None,conn=sddb.conn):
"""
TODO: if possible, remove this func and use get_dataset_() instead
"""
d=None
c = conn.cursor()
# Raise exception if having to much search keys
count=0
for va in (path,dataset_id,dataset_functional_id):
if va is not None:
count=count+1
if count>1:
raise SDException("SYNCDDAO-123","Too much arguments (path=%s,dataset_id=%s,dataset_functional_id=%s)"%(path,dataset_id,dataset_functional_id,))
if path is not None:
q="select * from dataset where path = '%s'" % path
elif dataset_id is not None:
q="select * from dataset where dataset_id = %i" % dataset_id
elif dataset_functional_id is not None:
q="select * from dataset where dataset_functional_id = '%s'" % dataset_functional_id
else:
raise SDException("SYNCDDAO-124","incorrect arguments")
c.execute(q)
rs=c.fetchone()
if rs is not None:
d=sdsqlutils.get_object_from_resultset(rs,Dataset)
c.close()
return d
def get_files_batch(conn=sddb.conn):
"""This method is used to loop over all files (note that we use yield here not to load all the rows in memory !!!)
Notes:
-
this method is like get_files_pagination, but use yield instead of using pagination
-
it seems not be possible to do write anywhere in the db file between two yields !!!!
-
(SELECT ... FOR UPDATE OF ... is not supported. This is understandable
considering the mechanics of SQLite in that row locking is redundant as
the entire database is locked when updating any bit of it)
-
if using same connection in select and insert, we get 'OperationalError: cannot commit transaction - SQL statements in progress'
if using different connection, we get 'OperationalError: database is locked'
"""
arraysize=100
c = conn.cursor()
q="select * from file order by file_id ASC"
c.execute(q)
for rs in large_query_helper(c,arraysize):
yield sdsqlutils.get_object_from_resultset(rs,File)
c.close()
def get_files(self):
files=[]
results=self.get_items()
for rs in results:
f=sdsqlutils.get_object_from_resultset(rs,sdtypes.File)
files.append(f)
return files
def get_files(q,type_,conn=sddb.conn):
files=[]
c = conn.cursor()
c.execute(q)
rs=c.fetchone()
while rs!=None:
files.append(sdsqlutils.get_object_from_resultset(rs,eval(type_))) # HACK: improve this
rs=c.fetchone()
c.close()
return files
def get_files(q, type_, conn=sddb.conn):
files = []
c = conn.cursor()
c.execute(q)
rs = c.fetchone()
while rs != None:
files.append(sdsqlutils.get_object_from_resultset(
rs, eval(type_))) # HACK: improve this
rs = c.fetchone()
c.close()
return files
def get_file(file_functional_id,conn=sddb.conn):
"""
notes
- returns None if file not found
- return type is File
"""
t=None
c = conn.cursor()
c.execute("select * from file where file_functional_id = ?", (file_functional_id,))
rs=c.fetchone()
if rs<>None:
t=sdsqlutils.get_object_from_resultset(rs,File)
c.close()
return t
def get_file(file_functional_id,conn=sddb.conn):
"""
notes
- returns None if file not found
- return type is File
"""
t=None
c = conn.cursor()
c.execute("select * from file where file_functional_id = ?", (file_functional_id,))
rs=c.fetchone()
if rs<>None:
t=sdsqlutils.get_object_from_resultset(rs,File)
c.close()
return t
def get_dataset_versions(i__d,i__compute_stats):
datasetVersions=DatasetVersions()
c = sddb.conn.cursor()
c.execute("select * from dataset where path_without_version=?",(i__d.path_without_version,))
rs=c.fetchone()
while rs!=None:
l__d=sdsqlutils.get_object_from_resultset(rs,Dataset)
if i__compute_stats:
l__d.statistics=get_dataset_stats(l__d)
datasetVersions.add_dataset_version(l__d)
rs=c.fetchone()
c.close()
return datasetVersions
def get_dataset_versions(i__d, i__compute_stats):
datasetVersions = DatasetVersions()
c = sddb.conn.cursor()
c.execute("select * from dataset where path_without_version=?",
(i__d.path_without_version, ))
rs = c.fetchone()
while rs != None:
l__d = sdsqlutils.get_object_from_resultset(rs, Dataset)
if i__compute_stats:
l__d.statistics = get_dataset_stats(l__d)
datasetVersions.add_dataset_version(l__d)
rs = c.fetchone()
c.close()
return datasetVersions
def get_dataset(path=None,
dataset_id=None,
dataset_functional_id=None,
conn=sddb.conn):
"""
TODO: if possible, remove this func and use get_dataset_() instead
"""
d = None
c = conn.cursor()
# Raise exception if having to much search keys
count = 0
for va in (path, dataset_id, dataset_functional_id):
if va is not None:
count = count + 1
if count > 1:
raise SDException(
"SYNCDDAO-123",
"Too much arguments (path=%s,dataset_id=%s,dataset_functional_id=%s)"
% (
path,
dataset_id,
dataset_functional_id,
))
if path is not None:
q = "select * from dataset where path = '%s'" % path
elif dataset_id is not None:
q = "select * from dataset where dataset_id = %i" % dataset_id
elif dataset_functional_id is not None:
q = "select * from dataset where dataset_functional_id = '%s'" % dataset_functional_id
else:
raise SDException("SYNCDDAO-124", "incorrect arguments")
c.execute(q)
rs = c.fetchone()
if rs is not None:
d = sdsqlutils.get_object_from_resultset(rs, Dataset)
c.close()
return d
def get_files(limit=None, conn=sddb.conn):
"""
Note
- if 'limit' is None, retrieve all records
"""
files = []
orderby = "file_functional_id"
limit_clause = "limit %i" % limit if limit is not None else ""
c = conn.cursor()
q = "select * from file order by %s desc %s" % (orderby, limit_clause)
c.execute(q)
rs = c.fetchone()
while rs != None:
files.append(sdsqlutils.get_object_from_resultset(rs, File))
rs = c.fetchone()
c.close()
return files
def get_files(limit=None,conn=sddb.conn):
"""
Note
- if 'limit' is None, retrieve all records
"""
files=[]
orderby="file_functional_id"
limit_clause="limit %i"%limit if limit is not None else ""
c = conn.cursor()
q="select * from file order by %s desc %s"%(orderby,limit_clause)
c.execute(q)
rs=c.fetchone()
while rs!=None:
files.append(sdsqlutils.get_object_from_resultset(rs,File))
rs=c.fetchone()
c.close()
return files
def get_events(limit=None,conn=sddb.conn,**search_constraints):
"""
Note
- one search constraint must be given at least
- if 'limit' is None, retrieve all records
"""
events=[]
search_placeholder=sdsqlutils.build_search_placeholder(search_constraints)
orderby="priority DESC, crea_date ASC"
limit_clause="limit %i"%limit if limit is not None else ""
c = conn.cursor()
q="select * from event where %s order by %s %s"%(search_placeholder,orderby,limit_clause)
c.execute(q,search_constraints)
rs=c.fetchone()
while rs!=None:
events.append(sdsqlutils.get_object_from_resultset(rs,Event))
rs=c.fetchone()
c.close()
return events
def get_files(limit=None,conn=sddb.conn,**search_constraints): # don't change arguments order here
"""
Notes
- one search constraint must be given at least
- if 'limit' is None, retrieve all records matching the search constraints
"""
files=[]
search_placeholder=sdsqlutils.build_search_placeholder(search_constraints)
orderby="priority DESC, checksum"
limit_clause="limit %i"%limit if limit is not None else ""
c = conn.cursor()
q="select * from file where %s order by %s %s"%(search_placeholder,orderby,limit_clause)
c.execute(q,search_constraints)
rs=c.fetchone()
while rs!=None:
files.append(sdsqlutils.get_object_from_resultset(rs,File))
rs=c.fetchone()
c.close()
return files
def get_events(limit=None, conn=sddb.conn, **search_constraints):
"""
Note
- one search constraint must be given at least
- if 'limit' is None, retrieve all records
"""
events = []
search_placeholder = sdsqlutils.build_search_placeholder(
search_constraints)
orderby = "priority DESC, crea_date ASC"
limit_clause = "limit %i" % limit if limit is not None else ""
c = conn.cursor()
q = "select * from event where %s order by %s %s" % (search_placeholder,
orderby, limit_clause)
c.execute(q, search_constraints)
rs = c.fetchone()
while rs != None:
events.append(sdsqlutils.get_object_from_resultset(rs, Event))
rs = c.fetchone()
c.close()
return events
def get_files_batch(conn=sddb.conn):
"""This method is used to loop over all files (note that we use yield here not to load all the rows in memory !)
Notes:
- this method is like get_files_pagination, but use yield instead of using pagination
- it seems not be possible to do write anywhere in the db file between two yields !!!!
-
(SELECT ... FOR UPDATE OF ... is not supported. This is understandable
considering the mechanics of SQLite in that row locking is redundant as
the entire database is locked when updating any bit of it)
-
- if using same connection in select and insert, we get 'OperationalError: cannot commit transaction - SQL statements in progress'
- if using different connection, we get 'OperationalError: database is locked'
"""
arraysize = 100
c = conn.cursor()
q = "select * from file order by file_id ASC"
c.execute(q)
for rs in large_query_helper(c, arraysize):
yield sdsqlutils.get_object_from_resultset(rs, File)
c.close()
def get_files(limit=None,conn=sddb.conn,**search_constraints): # don't change arguments order here
"""
Notes
- one search constraint must be given at least
- if 'limit' is None, retrieve all records matching the search constraints
"""
gfs0 = SDTimer.get_time()
data_node = search_constraints.get('data_node',None)
# Is the cache applicable to this situation? Then use it, unless it's empty.
if set(search_constraints.keys())==set(['status','data_node']) and\
search_constraints['status']==sdconst.TRANSFER_STATUS_WAITING and\
limit==1 and\
sdconst.GET_FILES_CACHING:
# ...limit==1 isn't essential, but the present coding is for limit==1
use_cache = True
if len( get_files.files.get( data_node, [] ) )>0:
gfs1 = SDTimer.get_elapsed_time( gfs0, show_microseconds=True )
sdlog.info("SDFILDAO-200","get_files time is %s, used cache, data_node %s"%\
(gfs1,data_node))
return [ get_files.files[data_node].pop(0) ]
else:
cachelen = 100
limit += cachelen
else:
use_cache = False
files=[]
c = conn.cursor()
search_placeholder=sdsqlutils.build_search_placeholder(search_constraints)
limit_clause="limit %i"%limit if limit is not None else ""
getfirst = priority_clause( data_node, use_cache, c )
if getfirst=='':
return []
q="select * from file where %s %s %s"%(search_placeholder,getfirst,limit_clause)
c.execute(q,search_constraints)
rs=c.fetchone()
#if use_cache and (rs is None or len(rs)==0):
if use_cache and (rs is None or len(rs)==0):
# No files found. Possibly we could find one if we retry at another priority level.
# Note that it makes no sense to do this if getfirst==''. That has alread triggered
# an early return.
highest_waiting_priority( data_node, c ) # compute the priority to retry at
getfirst = priority_clause( data_node, use_cache, c )
q="select * from file where %s %s %s"%(search_placeholder,getfirst,limit_clause)
c.execute(q,search_constraints)
rs = c.fetchone()
while rs!=None:
files.append(sdsqlutils.get_object_from_resultset(rs,File))
rs=c.fetchone()
c.close()
if len(files)>1 and use_cache:
# We shall return one of the files and cache the rest.
get_files.files[ data_node ] = files[1:]
files = files[0:1]
gfs1 = SDTimer.get_elapsed_time( gfs0, show_microseconds=True )
sdlog.info("SDFILDAO-200","get_files time is %s, search %s with %s"%(gfs1,q,search_constraints))
return files
