def getGroupsOnCategories(categories, callback):
groups = []
metadataAttrNameRefMonth = UUMETADATASTORAGEMONTH + '%0*d' % (2, datetime.now().month)
for category in categories:
iter = genquery.row_iterator(
"USER_NAME",
"USER_TYPE = 'rodsgroup' AND META_USER_ATTR_NAME = 'category' AND META_USER_ATTR_VALUE = '" + category + "' ",
genquery.AS_LIST, callback
)
for row in iter:
groupName = row[0]
iter2 = genquery.row_iterator(
"META_USER_ATTR_VALUE, USER_NAME, USER_GROUP_NAME",
"META_USER_ATTR_NAME = '" + metadataAttrNameRefMonth + "' AND USER_NAME = '" + groupName + "'",
genquery.AS_LIST, callback
)
data_size = 0
for row in iter2:
data = row[0]
temp = json.loads(data)
data_size = data_size + int(temp[2]) # no construction for summation required in this case
groups.append([groupName, data_size])
return groups
def pep_database_set_avu_metadata_pre(rule_args, callback, rei):
# callback.writeLine("serverLog", "pep_database_set_avu_metadata_pre. Arguments: " + str(len(rule_args)))
object_name = rule_args[4]
object_type = rule_args[3]
object_unit = rule_args[7]
object_attribute = rule_args[5]
# This policy is not using the helper allowAvuChange function as the set operation can also modify units indirectly
# Get all AVUs with attribute $id
fields = getFieldsForType(callback, object_type, object_name)
fields_id = fields['WHERE'] + " AND %s = '$id'" % (fields['a'])
rows = genquery.row_iterator([fields['a'], fields['v'], fields['u']], fields_id, genquery.AS_DICT, callback)
# From these AVUs extract the unit (root)
root_list = []
for row in rows:
root_list.append(row[fields['u']])
# Get the unit from the avu that is currently added.
for root in root_list:
# If the unit start with one of the roots, disallow the operation
if str(object_unit).startswith(root + "_"):
callback.msiOprDisallowed()
# A set operation can also change the unit of existing attributes
fields_a = fields['WHERE'] + " AND %s = '%s'" % (fields['a'], object_attribute)
rows = genquery.row_iterator([fields['a'], fields['v'], fields['u']], fields_a, genquery.AS_DICT, callback)
for row in rows:
for root in root_list:
# If the unit start with one of the roots, disallow the operation
if str(row[fields['u']]).startswith(root + "_"):
callback.msiOprDisallowed()
def getCategory(callback, rods_zone, group_name):
category = '-1'
schemaCategory = 'default'
# Find out category based on current group_name.
iter = genquery.row_iterator(
"META_USER_ATTR_NAME, META_USER_ATTR_VALUE",
"USER_GROUP_NAME = '" + group_name + "' AND META_USER_ATTR_NAME like 'category'",
genquery.AS_LIST, callback
)
for row in iter:
category = row[1]
if category != '-1':
# Test whether found category actually has a collection with XSD's.
# If not, fall back to default schema collection. Otherwise use category schema collection
# /tempZone/yoda/schemas/default
# - metadata.xsd
# - vault.xsd
xsdCollectionName = '/' + rods_zone + '/yoda/schemas/' + category
iter = genquery.row_iterator(
"COLL_NAME",
"DATA_NAME like '%%.xsd' AND COLL_NAME = '" + xsdCollectionName + "'",
genquery.AS_LIST, callback
)
for row in iter:
schemaCategory = category # As collection is present, the schemaCategory can be assigned the category
return schemaCategory
def get_first_eligible_input(callback,
input_colln,
task_id,
sort_key_func=None):
METADATA_TAG = Metadata_Tag
# Start with a set of all data objects in the input collection
# subtract from the set all objects with nonzero task id's set in iRODS metadata
eligible_inputs = set(
"{COLL_NAME}/{DATA_NAME}".format(**d) for d in \
row_iterator( ["COLL_NAME","DATA_NAME"],
"COLL_NAME = '{input_colln}'".format(**locals()),
AS_DICT,callback )
)-set(
"{COLL_NAME}/{DATA_NAME}".format(**d) for d in \
row_iterator( ["COLL_NAME","DATA_NAME"],
"COLL_NAME = '{input_colln}' and META_DATA_ATTR_NAME = '{METADATA_TAG}' "
"and META_DATA_ATTR_VALUE != '' ".format(**locals()),
AS_DICT,callback )
)
if sort_key_func:
eligible_inputs = sorted(list(eligible_inputs), key=sort_key_func)
else:
eligible_inputs = list(eligible_inputs)
chosen_input = None
if len(eligible_inputs):
chosen_input = eligible_inputs[0]
meta_stamp(callback, chosen_input, task_id)
return chosen_input
def getUnpreservableFiles(callback, rei, folder, list):
zoneName = ""
rods_zone = session_vars.get_map(rei)["client_user"]["irods_zone"]
# Retrieve JSON list of preservable file formats.
json = parseJson(callback,
"/" + rods_zone + "/yoda/file_formats/" + list + ".json")
preservableFormats = json['formats']
unpreservableFormats = []
# Retrieve all files in collection.
iter = genquery.row_iterator("DATA_NAME, COLL_NAME",
"COLL_NAME like '%s%%'" % (folder),
genquery.AS_LIST, callback)
for row in iter:
filename, file_extension = os.path.splitext(row[0])
# Convert to lowercase and remove dot.
file_extension = (file_extension.lower())[1:]
# Check if extention is in preservable format list.
if (file_extension not in preservableFormats):
unpreservableFormats.append(file_extension)
# Remove duplicate file formats.
output = []
for x in unpreservableFormats:
if x not in output:
output.append(x)
return {'formats': output}
def iiCheckMetadataXmlForSchemaIdentifier(rule_args, callback, rei):
rods_zone = session_vars.get_map(rei)["client_user"]["irods_zone"]
callback.writeString("stdout", "[METADATA] Start check for schema identifiers.\n")
# Find all research and vault collections, ordered by COLL_ID.
iter = genquery.row_iterator(
"ORDER(COLL_ID), COLL_NAME",
"COLL_NAME like '/%s/home/%%' AND DATA_NAME like 'yoda-metadata%%xml'" % (rods_zone),
genquery.AS_LIST, callback
)
# Check each collection in batch.
for row in iter:
coll_id = int(row[0])
coll_name = row[1]
pathParts = coll_name.split('/')
group_name = pathParts[3]
if 'research-' in group_name:
checkMetadataXmlForSchemaIdentifier(callback, rods_zone, coll_name, group_name, "yoda-metadata.xml")
elif 'vault-' in group_name:
# Get vault package path.
vault_package = '/'.join(pathParts[:5])
data_name = getLatestVaultMetadataXml(callback, vault_package)
if data_name != "":
checkMetadataXmlForSchemaIdentifier(callback, rods_zone, vault_package, group_name, data_name)
else:
callback.writeLine("stdout", "Missing metadata file: %s" % (vault_package))
callback.writeString("stdout", "[METADATA] Finished check for schema identifiers.\n")
def pep_database_del_avu_metadata_pre(rule_args, callback, rei):
# callback.writeLine("serverLog", "pep_database_del_avu_metadata_pre. Arguments: " + str(len(rule_args)))
object_name = rule_args[5]
object_type = rule_args[4]
object_attribute = rule_args[6]
object_value = rule_args[7]
object_unit = rule_args[8]
if not allowAvuChange(object_name, object_type, object_unit, callback):
callback.msiOprDisallowed()
# Wild card removal check
if "%" in [object_attribute, object_value, object_unit]:
# Get all AVU for the from object
fields = getFieldsForType(callback, object_type, object_name)
fields_a = fields['WHERE']
# if a,v or u is not wild card add to filter
if object_attribute != "%":
fields_a = fields_a + " AND %s = '%s'" % (fields['a'], object_attribute)
if object_value != "%":
fields_a = fields_a + " AND %s = '%s'" % (fields['v'], object_value)
if object_unit != "%":
fields_a = fields_a + " AND %s = '%s'" % (fields['u'], object_unit)
avus = genquery.row_iterator([fields['a'], fields['v'], fields['u']], fields_a,
genquery.AS_DICT, callback)
for avu in avus:
unit = str(avu[fields['u']])
if not allowAvuChange(object_name, object_type, unit, callback):
callback.msiOprDisallowed()
def uuRuleGetMonthStoragePerTierForGroup(rule_args, callback, rei):
groupName = rule_args[0]
currentMonth = int(rule_args[1]) # this is the month that came from the frontend
allStorage = [] # list of all month-tier combinations present including their storage size
# per month gather month/tier/storage information from metadata:
# metadata-attr-name = UUMETADATASTORAGEMONTH + '01'...'12'
# metadata-attr-val = [category,tier,storage] ... only tier and storage required wihtin this code
for counter in range(0, 11):
referenceMonth = currentMonth - counter
if referenceMonth < 1:
referenceMonth = referenceMonth + 12
metadataAttrNameRefMonth = UUMETADATASTORAGEMONTH + '%0*d' % (2, referenceMonth)
iter = genquery.row_iterator(
"META_USER_ATTR_VALUE, USER_NAME, USER_GROUP_NAME",
"META_USER_ATTR_NAME = '" + metadataAttrNameRefMonth + "' AND USER_NAME = '" + groupName + "'",
genquery.AS_LIST, callback
)
for row in iter:
data = json.loads(row[0])
tierName = data[1]
data_size = data[2] # no construction for summation required in this case
key = 'month=' + str(referenceMonth) + '-tier=' + tierName
allStorage.append({key: data_size})
rule_args[2] = json.dumps(allStorage)
def user_id_for_name(callback, username):
user_id=""
for i in row_iterator( 'USER_ID,USER_NAME', "USER_NAME = '{}'".format(username), AS_DICT, callback):
if 0 == len(user_id):
user_id = i['USER_ID']
return user_id
def this_host_tied_to_resc(callback, resc ):
import socket
this_host = socket.gethostname()
tied_host = ""
for rescvault in row_iterator( 'RESC_LOC',"RESC_NAME = '{resc}'".format(**locals()), AS_DICT,callback):
tied_host = rescvault['RESC_LOC']
return this_host == tied_host
def user_has_access (callback, rei, username, access_type_name, data_object_path='', collection_path=''):
access = False
access_types = { 'write':'1120', 'read':'1050', 'own':'1200' }
user_ID = user_id_for_name (callback,username)
do_query = (user_ID != '')
if data_object_path and not collection_path :
coll_name , data_name = split_irods_path (data_object_path)
condition = "DATA_NAME = '{0}' and COLL_NAME = '{1}' "\
"and DATA_ACCESS_USER_ID = '{2}' and DATA_ACCESS_TYPE >= '{3}'".format(
data_name, coll_name, user_ID, access_types[access_type_name] )
elif collection_path and not data_object_path:
condition = "COLL_NAME = '{0}' and COLL_ACCESS_USER_ID = '{1}' and COLL_ACCESS_TYPE >= '{2}' ".format(
collection_path, user_ID, access_types[access_type_name] )
else :
do_query = False
if do_query:
for i in row_iterator( "COLL_NAME", condition, AS_LIST, callback):
access = True
return access
def getSubcategories(callback, category):
categories = set() # Unique subcategories.
groupCategories = {} # Group name => { category => .., subcategory => .. }
# Collect metadata of each group into `groupCategories` until both
# the category and subcategory are available, then add the subcategory
# to `categories` if the category name matches.
iter = genquery.row_iterator(
"USER_GROUP_NAME, META_USER_ATTR_NAME, META_USER_ATTR_VALUE",
"USER_TYPE = 'rodsgroup' AND META_USER_ATTR_NAME LIKE '%category'",
genquery.AS_LIST, callback
)
for row in iter:
group = row[0]
key = row[1]
value = row[2]
if group not in groupCategories:
groupCategories[group] = {}
if key in ['category', 'subcategory']:
groupCategories[group][key] = value
if ('category' in groupCategories[group] and
'subcategory' in groupCategories[group]):
# Metadata complete, now filter on category.
if groupCategories[group]['category'] == category:
# Bingo, add to the subcategory list.
categories.add(groupCategories[group]['subcategory'])
del groupCategories[group]
return list(categories)
def replicate_data_objects( rule_args , callback , rei):
from_object = rule_args[0]
to_resource = rule_args[1]
from_resource = rule_args[2]
trim_after_replication = rule_args[3]
if len(to_resource) == 0 or len(from_object) == 0: return
path,obj = split_irods_path (from_object)
condition = "COLL_NAME = '{0}' and DATA_NAME = '{1}' ".format(path,obj)
if from_resource:
condition += " and DATA_RESC_NAME = '{}' ".format(from_resource)
data_objects = list(row_iterator('DATA_NAME,COLL_NAME,DATA_RESC_NAME,DATA_REPL_NUM', condition, AS_DICT, callback))
if not(data_objects):
condition = "COLL_NAME = '{0}' || like '{0}/%' " .format (from_object)
if from_resource:
condition += " and DATA_RESC_NAME = '{}'".format(from_resource)
data_objects = list(row_iterator('DATA_NAME,COLL_NAME,DATA_RESC_NAME,DATA_REPL_NUM', condition, AS_DICT, callback))
replicated = {}
for dobj in data_objects:
full_path = "{COLL_NAME}/{DATA_NAME}".format(**dobj)
if dobj['DATA_RESC_NAME'] == to_resource:
replicated[full_path] = True
else:
old_replication_status = replicated.get(full_path, False)
if not old_replication_status:
#callback.writeLine("stderr", "replicating: \n" + pprint.pformat(dobj))
retval = callback.msiDataObjRepl( full_path, "destRescName={0}".format(to_resource),0)
new_replication_status = retval['status']
replicated [full_path] = new_replication_status
if new_replication_status and not(old_replication_status) and trim_after_replication and \
dobj['DATA_RESC_NAME'] == from_resource != "":
trim_retval = callback.msiDataObjTrim( "{COLL_NAME}/{DATA_NAME}".format(**dobj), "null",
dobj['DATA_REPL_NUM'], "1", "null", 0)
def getUserNameFromUserId(callback, user_id):
user_name = ""
iter = genquery.row_iterator(
"USER_NAME",
"USER_ID = '%s'" % (str(user_id)),
genquery.AS_LIST, callback
)
for row in iter:
user_name = row[0]
return user_name
def getDataObjSize(callback, coll_name, data_name):
data_size = 0
iter = genquery.row_iterator(
"DATA_SIZE",
"COLL_NAME = '%s' AND DATA_NAME = '%s'" % (coll_name, data_name),
genquery.AS_LIST, callback
)
for row in iter:
data_size = row[0]
return data_size
def getCategories(callback):
categories = []
iter = genquery.row_iterator(
"META_USER_ATTR_VALUE",
"USER_TYPE = 'rodsgroup' AND META_USER_ATTR_NAME = 'category'",
genquery.AS_LIST, callback
)
for row in iter:
categories.append(row[0])
return categories
def data_object_physical_path_in_vault(callback, path, resc, force_creation_on_resc, vault_validate = None):
#pr = make_logger(callback,'stderr')
colln, dataobj = split_irods_path( path )
status = _data_object_exists_targeting_resc( callback, resc, colln, dataobj )
if status != 'already-exists' and force_creation_on_resc.upper() not in ('', 'N','NO','0') :
close_rv = {} ; repl_rv = {} ; close_rv = {}
if (status == 'need-repl'):
repl_rv = callback.msiDataObjRepl(path, "destRescName={}".format(resc), 0)
#pr("repl status = %r arg-retn %r " % (repl_rv['status'],repl_rv['arguments'][2]))
else:
create_rv = callback.msiDataObjCreate(path, "forceFlag=++++destRescName={}".format(resc), 0)
descriptor = create_rv['arguments'][2]
if type(descriptor) is int and descriptor > 0:
close_rv = callback.msiDataObjClose(descriptor,0)
#pr("close(%r) status = %r arg-retn %r " % (descriptor,close_rv['status'],close_rv['arguments'][1]))
v = {}
if type(vault_validate) is dict: # - get vault path to match against data object phys. path
v = vault_validate
leading_path = ""
for rescvault in row_iterator( 'RESC_VAULT_PATH',"RESC_NAME = '{resc}'".format(**locals()), AS_DICT,callback ):
leading_path = rescvault['RESC_VAULT_PATH']
phys_path = ''
for p in row_iterator("DATA_PATH",
"DATA_RESC_NAME = '{resc}' and DATA_NAME = '{dataobj}' and COLL_NAME = '{colln}' ".format(**locals()),
AS_DICT,callback):
phys_path = p['DATA_PATH']
if leading_path != '' and phys_path.startswith( leading_path ):
v['vault_relative_path'] = phys_path [len(leading_path):].lstrip('/')
return leading_path
def pep_database_copy_avu_metadata_pre(rule_args, callback, rei):
# callback.writeLine("serverLog", "pep_database_copy_avu_metadata_pre. Arguments: " + str(len(rule_args)))
object_name_from = rule_args[5]
object_type_from = rule_args[3]
object_name_to = rule_args[6]
object_type_to = rule_args[4]
# Get all AVU for the from object
fields_from = getFieldsForType(callback, object_type_from, object_name_from)
avus_from = genquery.row_iterator([fields_from['a'], fields_from['v'], fields_from['u']], fields_from['WHERE'],
genquery.AS_DICT, callback)
# Get all AVUs with attribute $id from the to object
fields_to = getFieldsForType(callback, object_type_to, object_name_to)
fields_id = fields_to['WHERE'] + " AND %s = '$id'" % (fields_to['a'])
rows = genquery.row_iterator([fields_to['a'], fields_to['v'], fields_to['u']], fields_id, genquery.AS_DICT,
callback)
# From these AVUs extract the unit (root)
root_list_to = []
for row in rows:
root_list_to.append(row[fields_to['u']])
# Regular expression pattern for unit field
pattern = re.compile(jsonavu.RE_UNIT)
# For all AVUs on the from object check if one starts with the one of the root from the to object
for avu in avus_from:
for root in root_list_to:
# Match unit to extract all info
unit = str(avu[fields_from['u']])
# If unit is matching
if pattern.match(unit) and unit.startswith(root + "_"):
# callback.writeLine("serverLog", "JSON root " + root + " is already in use in the to object")
# callback.msiExit("-1101000", "JSON root " + root + " is already in use in the to object")
callback.msiOprDisallowed()
def getTierOnResourceName(resourceName, callback):
tierName = UUDEFAULTRESOURCETIER # Add default tier as this might not be present in database.
# find (possibly present) tier for this resource
iter = genquery.row_iterator(
"RESC_ID, RESC_NAME, META_RESC_ATTR_NAME, META_RESC_ATTR_VALUE",
"RESC_NAME = '" + resourceName + "' AND META_RESC_ATTR_NAME = '" + UURESOURCETIERATTRNAME + "'",
genquery.AS_LIST, callback
)
for row in iter:
tierName = row[3]
return tierName
def getProvenanceLog(callback, folder):
provenance_log = []
# Retrieve all provenance logs on a folder.
iter = genquery.row_iterator(
"order(META_COLL_ATTR_VALUE)",
"COLL_NAME = '%s' AND META_COLL_ATTR_NAME = 'org_action_log'" %
(folder), genquery.AS_LIST, callback)
for row in iter:
log_item = json.loads(row[0])
provenance_log.append(log_item)
return provenance_log
def getCategoriesDatamanager(datamanagerName, callback):
categories = []
iter = genquery.row_iterator(
"USER_NAME",
"USER_TYPE = 'rodsgroup' AND USER_NAME like 'datamanager-%'",
genquery.AS_LIST, callback
)
for row in iter:
# @TODO membership still has to be checked
datamanagerGroupname = row[0]
temp = datamanagerGroupname.split('-') # 'datamanager-initial' is groupname of datamanager, second part is category
categories.append(temp[1])
return categories
def _list_all_application_configs(callback, ctx, save_rows = None, as_lookup=False):
# TODO : renaming to irods::compute_to_data
rows = scratch = []
if isinstance(save_rows,list): rows = save_rows
elif isinstance(save_rows,tuple): rows = list(save_rows)
if (rows is scratch) or (rows is save_rows):
rows[:] = [ r for r in row_iterator(
"COLL_NAME,DATA_NAME,META_DATA_ATTR_VALUE",
("DATA_NAME like '%.json' "
"and COLL_NAME like '%/configured_applications' "
"and COLL_NAME not like '%/trash/%' "
"and META_DATA_ATTR_NAME = 'irods::compute::application' "
"and META_DATA_ATTR_VALUE like '_%' "
# "and META_DATA_ATTR_UNITS like 'docker' "
).format(**ctx) , AS_DICT, callback) ]
return [ "{COLL_NAME}/{DATA_NAME}".format(**row) for row in rows ] if not as_lookup else \
{ row["META_DATA_ATTR_VALUE"]:"{COLL_NAME}/{DATA_NAME}".format(**row) for row in rows }
def getLatestVaultMetadataXml(callback, vaultPackage):
dataName = ""
iter = genquery.row_iterator(
"DATA_NAME, DATA_SIZE",
"COLL_NAME = '" + vaultPackage + "' AND DATA_NAME like 'yoda-metadata[%].xml'",
genquery.AS_LIST, callback
)
# Loop through all XMLs.
for row in iter:
data_name = row[0]
data_size = int(row[1])
if dataName == "" or (dataName < data_name and len(dataName) <= len(data_name)):
dataName = data_name
return dataName
def uuRuleGetResourcesAndTierData(rule_args, callback, rei):
resourceList = list()
iter = genquery.row_iterator(
"RESC_ID, RESC_NAME",
"",
genquery.AS_LIST, callback
)
for row in iter:
resourceId = row[0]
resourceName = row[1]
tierName = getTierOnResourceName(resourceName, callback)
resourceList.append({'resourceName': resourceName,
'resourceId': resourceId,
'org_storage_tier': tierName})
rule_args[0] = json.dumps(resourceList)
def pep_api_phy_path_reg_post(rule_args,callback,rei):
pr = p(callback)
pr( str(rule_args) )
pr("REG ** post ** {!r}".format(rule_args))
# - get file path and resc name for potentially remote netcdf extraction
phyPath = ""
rescName = ""
cndI = rule_args[2].condInput
for i in range(cndI.len):
if cndI.key[i] == 'filePath': phyPath = cnd.value[i]
elif cndI.key[i] == 'destRescName': rescName = cnd.value[i]
lgPath = str(rule_args[2].objPath)
lgPath_host = ""
for x in row_iterator('RESC_LOC' , "RESC_NAME = '{}'".format(rescName), AS_LIST, callback):
logPath_host= x[0]
def uuRuleExportMonthlyCategoryStatisticsDM(rule_args, callback, rei):
datamanagerUser = rule_args[0]
categories = getCategoriesDatamanager(datamanagerUser, callback)
allStorage = []
# Select a full year by not limiting UUMETADATASTORAGEMONTH to a perticular month. But only on its presence.
# There always is a maximum of one year of history of storage data
for category in categories:
groupToSubcategory = {}
iter = genquery.row_iterator(
"META_USER_ATTR_VALUE, META_USER_ATTR_NAME, USER_NAME, USER_GROUP_NAME",
"META_USER_ATTR_VALUE like '[\"" + category + "\",%' AND META_USER_ATTR_NAME like '" + UUMETADATASTORAGEMONTH + "%'",
genquery.AS_LIST, callback
)
for row in iter:
attrValue = row[0]
month = row[1]
month = str(int(month[-2:])) # the month storage data is about, is taken from the attr_name of the AVU
groupName = row[3]
# Determine subcategory on groupName
try:
subcategory = groupToSubcategory[groupName]
except KeyError:
catInfo = groupGetCategoryInfo(groupName, callback)
subcategory = catInfo['subcategory']
groupToSubcategory[groupName] = subcategory
temp = json.loads(attrValue)
category = temp[0]
tier = temp[1]
storage = int(temp[2])
allStorage.append({'category': category,
'subcategory': subcategory,
'groupname': groupName,
'tier': tier,
'month': month,
'storage': str(storage)})
rule_args[1] = json.dumps(allStorage)
def setJsonSchemaToObj(rule_args, callback, rei):
"""
This rule stores a given JSON-schema as AVU's to an object
:param rule_args:
Argument 0: The object name (/nlmumc/P000000003, /nlmumc/projects/metadata.xml, [email protected], demoResc)
Argument 1: The object type
-d for data object
-R for resource
-C for collection
-u for user
Argument 2: URL to the JSON-Schema example https://api.myjson.com/bins/17vejk
Argument 3: The JSON root according to https://github.com/MaastrichtUniversity/irods_avu_json.
:param callback:
:param rei:
:return:
"""
object_name = rule_args[0]
object_type = rule_args[1]
json_schema_url = rule_args[2]
json_root = rule_args[3]
# Check if this root has been used before
fields = getFieldsForType(callback, object_type, object_name)
avus = genquery.row_iterator([fields['a'], fields['v'], fields['u']], fields['WHERE'], genquery.AS_DICT, callback)
# Regular expression pattern for unit field
pattern = re.compile(jsonavu.RE_UNIT)
for avu in avus:
# Match unit to extract all info
unit = str(avu[fields['u']])
# If unit is matching
if pattern.match(unit) and unit.startswith(json_root + "_"):
callback.msiExit("-1101000", "JSON root " + json_root + " is already in use")
# Delete existing $id AVU for this JSON root
callback.msi_rmw_avu(object_type, object_name, '$id', "%", json_root)
# Set new $id AVU
callback.msi_add_avu(object_type, object_name, '$id', json_schema_url, json_root)
def getMonthlyCategoryStorageStatistics(categories, callback):
month = '%0*d' % (2, datetime.now().month)
metadataName = UUMETADATASTORAGEMONTH + month
storageDict = {}
for category in categories:
iter = genquery.row_iterator(
"META_USER_ATTR_VALUE, META_USER_ATTR_NAME, USER_NAME, USER_GROUP_NAME",
"META_USER_ATTR_VALUE like '[\"" + category + "\",%' AND META_USER_ATTR_NAME = '" + metadataName + "'",
genquery.AS_LIST, callback
)
for row in iter:
# hier wordt door alle groepen gezocht, geordend van een category.
# per tier moet worden gesommeerd om totale hoeveelheid storage op een tier te verkrijgen.
attrValue = row[0]
temp = json.loads(attrValue)
category = temp[0]
tier = temp[1]
storage = int(float(temp[2]))
try:
storageDict[category][tier] = storageDict[category][tier] + storage
except KeyError:
callback.writeString('serverLog', 'Exception')
# if key error, can be either category or category/tier combination is missing
try:
storageDict[category][tier] = storage
except KeyError:
storageDict[category] = {tier: storage}
# prepare for json output, convert storageDict into dict with keys
allStorage = []
for category in storageDict:
for tier in storageDict[category]:
allStorage.append({'category': category,
'tier': tier,
'storage': str(storageDict[category][tier])})
return json.dumps(allStorage)
def checkDataObjectIntegrity(callback, data_id):
# Obtain all replicas of a data object.
iter = genquery.row_iterator(
"DATA_ID, DATA_NAME, DATA_SIZE, DATA_CHECKSUM, COLL_NAME, RESC_VAULT_PATH, RESC_LOC",
"DATA_ID = '%s'" % data_id, genquery.AS_LIST, callback)
# Loop through all replicas.
for row in iter:
data_object = DataObject._make(
[row[0], row[1], row[2], row[3], row[4], row[5], row[6]])
# Build file path to data object.
coll_name = os.path.join(
*(data_object.coll_name.split(os.path.sep)[2:]))
file_path = data_object.resc_path + "/" + coll_name + "/" + data_object.name
# Check integrity on the resource.
remote_rule = "checkDataObjectRemote('%s', '%s', '%s')" % \
(file_path.replace("'", "\\'"), data_object.size, data_object.checksum)
callback.remoteExec("%s" % data_object.resc_loc, "", remote_rule, "")
def groupGetCategoryInfo(groupName, callback):
category = ''
subcategory = ''
iter = genquery.row_iterator(
"META_USER_ATTR_NAME, META_USER_ATTR_VALUE",
"USER_GROUP_NAME = '" + groupName + "' AND META_USER_ATTR_NAME LIKE '%category'",
genquery.AS_LIST, callback
)
for row in iter:
attrName = row[0]
attrValue = row[1]
if attrName == 'category':
category = attrValue
elif attrName == 'subcategory':
subcategory = attrValue
return {'category': category, 'subcategory': subcategory}
