def get_tree_root(node, irods_user, filters, allowed_collection_ids, root):
ids = {}
thumbs_ids_collection = None
if allowed_collection_ids is not None:
with TuRODSSession(irods_user) as session:
coll = session.collections.get(node)
my_root = Node(node, root, id=coll.id, path=node, expanded=False)
for path in sorted(allowed_collection_ids.keys()):
splitted = path.split(node)
starting_root1 = node
my_parent = my_root
if len(splitted) == 2:
sub_folders = splitted[1][1:].split('/')
for x in range( len(sub_folders)):
p = starting_root1 + '/' + sub_folders[x]
if p in ids:
my_parent = ids[p]
else:
try:
coll = session.collections.get(p)
my_parent = Node(p, my_parent, path=p, id=coll.id, expanded=False)
ids[p] = my_parent
except Exception as e:
print('Failed', e)
pass
starting_root1 = p
return None, None
irods_user_id = get_irods_user_id(irods_user)
with TuRODSSession(irods_user) as session:
try:
coll = session.collections.get(node)
if is_allowed(allowed_collection_ids, coll.path):
sub = Node(node, root, id=coll.id, path=coll.path, expanded=False)
thumbs_ids_collection = None
#thumbs_ids_collection = search_objects(coll, sub, filters, irods_user_id)
for col in coll.subcollections:
if is_allowed(allowed_collection_ids, col.path):
Node(col.name, sub, id=col.id, path=col.path)
return thumbs_ids_collection, coll.id
except CAT_NO_ROWS_FOUND:
pass
except Exception as e:
log_and_raise_exception("Error in get_tree_root", e)
return thumbs_ids_collection, None
def permissions(folder, name):
users_dict = get_user_dict()
show_groups(users_dict)
with TuRODSSession() as session:
coll = session.collections.get(name)
permission = []
user_with_permissions = {}
with TuRODSSession() as session:
sql = "select user_id, access_type_id from R_OBJT_ACCESS WHERE object_id = {}".format(
coll.id)
columns = [CollectionAccess.user_id, CollectionAccess.access_id]
alias = 'get_permissions'
query = SpecificQuery(session, sql, alias, columns)
# register specific query in iCAT
_ = query.register()
exception = None
try:
for result in query.get_results():
usr_id = result[CollectionAccess.user_id]
access_id = result[CollectionAccess.access_id]
permission.append({
'user': users_dict[usr_id],
'access': permissions_dict[access_id],
'hasRights': True
})
user_with_permissions[usr_id] = usr_id
print(usr_id, users_dict[usr_id])
except CAT_NO_ROWS_FOUND:
pass
except Exception as e:
exception = e
_ = query.remove()
if exception is not None:
log_and_raise_exception('Error fetching permissions', exception)
for usr_id in users_dict:
if usr_id not in user_with_permissions:
permission.append({
'user': users_dict[usr_id],
'access': '',
'hasRights': False
})
return permission
def get_collection_id(name):
with TuRODSSession() as session:
query = session.query(Collection.id).filter(Collection.name == name)
result = query.execute()
if len(result) > 0:
return result[0][Collection.id]
return None
def delete_permission(user_id, object_id):
with TuRODSSession() as session:
sql = "delete from R_OBJT_ACCESS WHERE object_id = {} and user_id = {}".format(
object_id, 24746)
alias = 'del_permissions'
query = SpecificQuery(session, sql, alias)
# register specific query in iCAT
_ = query.register()
exception = None
try:
query.execute()
query.get_results()
except Exception as e:
exception = e
# delete specific query
_ = query.remove()
if exception is not None:
log_and_raise_exception('Error in delete_permission', exception)
def get_thumbnail_list(sql, thumbs_ids_collection):
with TuRODSSession() as session:
columns = [DataObject.id]
alias = 'get_thumbnail_list'
query = SpecificQuery(session, sql, alias, columns)
# register specific query in iCAT
_ = query.register()
exception = None
try:
results = query.get_results()
for result in results:
object_id = result[DataObject.id]
thumbs_ids_collection.append(object_id)
except CAT_NO_ROWS_FOUND:
pass
except Exception as e:
exception = e
# delete specific query
_ = query.remove()
if exception is not None:
log_and_raise_exception("Error in get_thumbnail_list", exception)
return thumbs_ids_collection
def remove_none(attribute_name):
with TuRODSSession() as session:
sql = "select meta_attr_value from R_META_MAIN where meta_attr_name = '{}'".format(attribute_name)
alias = 'remove_none'
columns = [CollectionMeta.value]
query = SpecificQuery(session, sql, alias, columns)
# register specific query in iCAT
_ = query.register()
exception = None
try:
for result in query.get_results():
print("Result: '{}'".format(result[CollectionMeta.value]))
except Exception as e:
exception = e
# delete specific query
_ = query.remove()
if exception is not None:
log_and_raise_exception('Error in remove_none', exception)
def get_permissions_dict():
permissions_dict = {}
with TuRODSSession() as session:
sql = "select token_id, token_name from R_TOKN_MAIN"
columns = [CollectionAccess.user_id, CollectionAccess.name]
alias = 'get_permissions_codes'
query = SpecificQuery(session, sql, alias, columns)
# register specific query in iCAT
_ = query.register()
exception = None
try:
for result in query.get_results():
token_id = result[CollectionAccess.user_id]
name = result[CollectionAccess.name]
permissions_dict[token_id] = name
except Exception as e:
exception = e
_ = query.remove()
if exception is not None:
log_and_raise_exception('Error in get_permissions_dict', exception)
return permissions_dict
def update_permissions(user, access, recursive, name):
access_type_id = get_access_type_id(access)
object_id = get_collection_id(name)
users_dict = get_user_dict()
user_id = list(users_dict.keys())[list(users_dict.values()).index(user)]
with TuRODSSession() as session:
sql = "update R_OBJT_ACCESS set access_type_id = {} where object_id = {} and user_id = {}" \
.format(access_type_id, object_id, user_id)
alias = 'update_permissions2'
query = SpecificQuery(session, sql, alias)
# register specific query in iCAT
_ = query.register()
exception = None
try:
query.execute()
except CAT_NO_ROWS_FOUND:
pass
except Exception as e:
exception = e
_ = query.remove()
if exception is not None:
log_and_raise_exception('Error in changing the permissions access_type_id = {} object_id = {} and user_id = {}' \
.format(access_type_id, object_id, user_id), exception)
def get_tree(node, irods_user, filters, allowed_collection_ids):
root = None
thumbs_ids_collection = None
irods_user_id = get_irods_user_id(irods_user)
with TuRODSSession(irods_user) as session:
try:
coll = session.collections.get(node)
if is_allowed(allowed_collection_ids, coll.path):
root = Node(coll.name, id=coll.id,path=coll.path, expanded=False)
thumbs_ids_collection = search_objects(coll, root, filters, irods_user_id)
for col in coll.subcollections:
if is_allowed(allowed_collection_ids, col.path):
Node(col.name, root, id=col.id, path=col.path)
if root is None:
return None, thumbs_ids_collection, coll.id
return root, thumbs_ids_collection, coll.id
except CAT_NO_ROWS_FOUND:
pass
except Exception as e:
log_and_raise_exception("Error in get_tree", e)
return root, thumbs_ids_collection, None
def has_objects(clauses, user, starting_folder):
coll_ids = {}
irods_user_id = get_irods_user_id(user.profile.irods_user)
with TuRODSSession() as session:
base_sql = "select distinct R_DATA_MAIN.coll_id, R_COLL_MAIN.coll_name from R_DATA_MAIN " \
"JOIN R_COLL_MAIN ON R_DATA_MAIN.coll_id = R_COLL_MAIN.coll_id " \
"JOIN R_OBJT_METAMAP r_data_metamap ON R_DATA_MAIN.data_id = r_data_metamap.object_id " \
"JOIN R_META_MAIN r_data_meta_main ON r_data_metamap.meta_id = r_data_meta_main.meta_id " \
"JOIN R_OBJT_ACCESS OA ON OA.object_id = R_DATA_MAIN.data_id " \
"WHERE user_id in ( select group_user_id from R_user_group where user_id = {0})".format(irods_user_id)
nb_of_filters = len(clauses)
columns = [DataObject.collection_id, DataObject.path]
if nb_of_filters == 1:
sql = base_sql + clauses[0]
else:
sql = ''
n = 0
for clause in clauses:
if n == 0:
sql = base_sql + clause
n = n + 1
else:
sql = sql + ' intersect ' + base_sql + clause
alias = 'list_data_name_idyy'
query = SpecificQuery(session, sql, alias, columns)
# register specific query in iCAT
_ = query.register()
exception = None
try:
for result in query.get_results():
collid = result[DataObject.collection_id]
path = result[DataObject.path]
for folder in starting_folder.split(';'):
if path.startswith(folder):
coll_ids[path] = collid
break
except CAT_NO_ROWS_FOUND:
pass
except Exception as e:
exception = e
_ = query.remove()
if exception is not None:
log_and_raise_exception('Error in has_objects',exception)
return coll_ids
def get_user_dict():
users_dict = {}
with TuRODSSession() as session:
query = session.query(IrodsUser)
for result in query.get_results():
users_dict[result[IrodsUser.id]] = result[IrodsUser.name]
return users_dict
def folders(irods_user, root):
folders = []
for rt in root.split(';'):
folders.append({'display': rt, 'value': rt})
with TuRODSSession(irods_user) as session:
coll = session.collections.get(rt)
folders_recursive(coll, 1, folders)
return folders
def get_irods_user_id(irods_user):
try:
with TuRODSSession() as session:
query = session.query(IrodsUser.name, IrodsUser.id).filter(IrodsUser.name == irods_user)
result = query.one()
return result[IrodsUser.id]
except Exception as e:
log_exception('Error in get_irods_user_id',e)
return None
def create_thumbnails():
lst = []
for usr in User.objects.all():
profile = usr.profile
if profile is not None:
if profile.create_thumbs is False:
continue
irods_user = profile.irods_user
root = profile.root
#root = '/tempZone/home/garys'
with TuRODSSession(irods_user) as session:
try:
coll = session.collections.get(root)
create_thumbnails_recursive(coll, lst)
except Exception as e:
log_and_raise_exception("Error in create_thumbnails", e)
'''
placed the TuRODSSession in the loop: when the file for some reason is not available anymore ( bug iRods? )
then a new connection would be made and the old one not be disconnected. By placing in the loop: after the with it will be closed!
'''
for entry in lst:
file_data = None
path = entry['path']
folder_id = entry['f_id']
with TuRODSSession(irods_user) as session:
print('Creating thumb for ', path)
obj = session.data_objects.get(path)
file_data = resize_picture_in_temp(obj, path)
if file_data is not None:
t = Thumbnail()
t.image = 'data:image/jpeg;base64,{}'.format(base64.b64encode(file_data.getvalue()).decode("ascii"))
t.id = obj.id
t.name = os.path.basename(obj.path)
t.folder_id = folder_id
t.save()
def get_distinct_object_metadata_attr_per_user(usr, irods_user_id):
with TuRODSSession() as session:
sql = "select distinct r_data_meta_main.meta_attr_name from R_DATA_MAIN " \
"JOIN R_OBJT_METAMAP r_data_metamap ON R_DATA_MAIN.data_id = r_data_metamap.object_id " \
"JOIN R_META_MAIN r_data_meta_main ON r_data_metamap.meta_id = r_data_meta_main.meta_id " \
"JOIN R_OBJT_ACCESS OA ON OA.object_id = R_DATA_MAIN.data_id " \
"where user_id in ( select group_user_id from R_user_group where user_id = {0}) order by r_data_meta_main.meta_attr_name".format(irods_user_id)
alias = 'list_data_name_id118'
columns = [DataObjectMeta.name]
query = SpecificQuery(session, sql, alias, columns)
# register specific query in iCAT
_ = query.register()
exception = None
new_att_list = []
try:
for result in query.get_results():
name = result[DataObjectMeta.name]
name_unique = name.replace(' ', '%20')
# if we have two attribute 'ColorMode' and 'ColorMode ' update_or_create does not see the difference @%$$%
uima, created = UserIrodsMetaDataAttribute.objects.update_or_create(
name_unique__exact=name_unique, object_or_collection=True, user=usr)
if created:
uima.name = name
uima.name_unique = name_unique
uima.save()
new_att_list.append(uima)
except CAT_NO_ROWS_FOUND:
pass
except Exception as e:
exception = e
# delete specific query
_ = query.remove()
if exception is not None:
log_and_raise_exception('Error in get_distinct_object_metadata_attr_per_user', exception)
#for uima in UserIrodsMetaDataAttribute.objects.filter(user=usr, object_or_collection=True):
for uima in new_att_list:
uima.type = determine_object_type(session, uima.name, irods_user_id)
if uima.type is None:
uima.delete()
else:
uima.save()
def testje(path, user_id):
with TuRODSSession() as session:
sql = "select object_id from R_OBJT_ACCESS where user_id = {} and object_id in" \
"(select R_COLL_MAIN.coll_id from R_COLL_MAIN WHERE R_COLL_MAIN.coll_name like '{}%' )". \
format(user_id, path)
sql = "select R_COLL_MAIN.coll_id from R_COLL_MAIN JOIN R_OBJT_ACCESS ON R_COLL_MAIN.coll_id = R_OBJT_ACCESS.object_id " \
"WHERE R_COLL_MAIN.coll_name like '{}% and R_OBJT_ACCESS.user_id = {}".format(path, user_id)
sql = "select R_COLL_MAIN.coll_id from R_COLL_MAIN " \
"JOIN R_OBJT_ACCESS OA ON OA.object_id = R_COLL_MAIN.coll_id " \
"WHERE user_id = " + str(user_id)
#sql = "select R_COLL_MAIN.coll_id from R_COLL_MAIN WHERE R_COLL_MAIN.coll_name like '{}%' ".format(path)
print(sql)
alias = 'blepper14'
columns = [Collection.id]
query = SpecificQuery(session, sql, alias, columns)
# register specific query in iCAT
_ = query.register()
exception = None
n = 0
try:
query.execute()
for result in query.get_results():
n = n + 1
print(n, result[Collection.id])
print(get_collection_name(result[Collection.id]))
except CAT_NO_ROWS_FOUND:
print('no exception')
except Exception as e:
exception = e
_ = query.remove()
if exception is not None:
log_and_raise_exception('Error', exception)
def set_metadata(irods_user, dict, folder):
path = dict['path']
data = {}
with TuRODSSession(irods_user) as session:
if folder:
obj = session.collections.get(path)
else:
obj = session.data_objects.get(path)
for m in obj.metadata.items():
if m.name in dict:
obj.metadata.remove(m)
for key in dict:
if ( key != 'path'):
if ( not key.endswith('-units')):
obj.metadata.add(key,dict[key], dict[key+'-units'])
return data
def get_metadata(irods_user, path, folder):
path = '/' + path
data = {}
with TuRODSSession(irods_user) as session:
metadata = []
if folder:
obj = session.collections.get(path)
else:
obj = session.data_objects.get(path)
for m in obj.metadata.items():
d = { 'avu_id' : m.avu_id, 'key' : m.name, 'units' : m.units, 'val' : m.value}
metadata.append(d)
data['metadata'] = metadata
sort_on = "key"
decorated = [(dict_[sort_on], dict_) for dict_ in metadata]
decorated.sort()
result = [dict_ for (key, dict_) in decorated]
data['metadata'] = result
return data
def show_groups(users_dict):
with TuRODSSession() as session:
query = session.query(UserGroup.id, UserGroup.name)
for result in query.get_results():
print(result[UserGroup.id], result[UserGroup.name])
sql = "select group_user_id, user_id from R_user_group"
alias = 'blepper15'
columns = [CollectionAccess.user_id, CollectionAccess.access_id]
query = SpecificQuery(session, sql, alias, columns)
# register specific query in iCAT
_ = query.register()
exception = None
n = 0
try:
query.execute()
for result in query.get_results():
n = n + 1
user = users_dict[result[CollectionAccess.access_id]]
#print(n,result[CollectionAccess.user_id], user)
print(result[CollectionAccess.user_id],
result[CollectionAccess.access_id])
except CAT_NO_ROWS_FOUND:
print('no exception')
except Exception as e:
exception = e
_ = query.remove()
if exception is not None:
log_and_raise_exception('Error', exception)
def put_collection(irods_user, collection):
with TuRODSSession(irods_user) as session:
session.collections.create('/' + collection)
def get_size(irods_user):
with TuRODSSession(irods_user) as session:
query = session.query(DataObject.owner_name).count(DataObject.id).sum(DataObject.size)
result = query.execute()
return result[0][DataObject.size]
return 0
def search_objects(coll, parent, filters, irods_user_id):
folder = coll.path
clauses = []
thumbs_ids_collection = []
with TuRODSSession() as session:
base_sql = "select R_DATA_MAIN.data_name, R_DATA_MAIN.data_id, R_DATA_MAIN.data_size from R_DATA_MAIN " \
"JOIN R_OBJT_METAMAP r_data_metamap ON R_DATA_MAIN.data_id = r_data_metamap.object_id " \
"JOIN R_META_MAIN r_data_meta_main ON r_data_metamap.meta_id = r_data_meta_main.meta_id " \
"JOIN R_OBJT_ACCESS OA ON OA.object_id = R_DATA_MAIN.data_id " \
"where R_DATA_MAIN.coll_id = {0} and user_id in (select group_user_id from R_user_group where user_id = {1})".format(coll.id, irods_user_id)
if filters is not None:
for key in filters.keys():
clause = create_obj_filter(key, filters[key])
if clause != None:
clauses.append(clause)
nb_of_filters = len(clauses)
columns = [DataObject.name, DataObject.id, DataObject.size]
if nb_of_filters == 0:
sql = "select R_DATA_MAIN.data_name, R_DATA_MAIN.data_id, R_DATA_MAIN.data_size from R_DATA_MAIN " \
"JOIN R_OBJT_ACCESS OA ON OA.object_id = R_DATA_MAIN.data_id " \
"where R_DATA_MAIN.coll_id = {0} and user_id in (select group_user_id from R_user_group where user_id = {1})".format(coll.id, irods_user_id)
elif nb_of_filters == 1:
sql = base_sql + clauses[0]
else:
sql = ''
n = 0
for clause in clauses:
if n == 0:
sql = base_sql + clause
n = n + 1
else:
sql = sql + ' intersect ' + base_sql + clause
alias = 'xx_list_data_name_id11'
query = SpecificQuery(session, sql, alias, columns)
# register specific query in iCAT
_ = query.register()
exception = None
try:
thumbs = []
results = query.get_results()
for result in results:
name = result[DataObject.name]
path = folder + "/" + name
id = result[DataObject.id]
size = result[DataObject.size]
filename, extension = os.path.splitext(name)
if is_thumbnail(extension):
thumbs.append(thumbs)
#th = Thumbnail.objects.filter(id=id).first()
#if th is not None:
thumbs_ids_collection.append(id)
Node(name, parent=parent, id=id, path=path, leaf=True, size=size)
except CAT_NO_ROWS_FOUND:
pass
except Exception as e:
exception = e
# delete specific query
_ = query.remove()
if exception is not None:
log_and_raise_exception("Error in search_objects", exception)
return thumbs_ids_collection
def get_collection_name(coll_id):
with TuRODSSession() as session:
query = session.query(Collection.name).filter(Collection.id == coll_id)
result = query.execute()
return result[0][Collection.name]
return None
def put_object(irods_user, local_file, irods_file):
with TuRODSSession(irods_user) as session:
session.data_objects.put( local_file, irods_file)