def getGroupsOnCategories(categories, callback): groups = [] metadataAttrNameRefMonth = UUMETADATASTORAGEMONTH + '%0*d' % (2, datetime.now().month) for category in categories: iter = genquery.row_iterator( "USER_NAME", "USER_TYPE = 'rodsgroup' AND META_USER_ATTR_NAME = 'category' AND META_USER_ATTR_VALUE = '" + category + "' ", genquery.AS_LIST, callback ) for row in iter: groupName = row[0] iter2 = genquery.row_iterator( "META_USER_ATTR_VALUE, USER_NAME, USER_GROUP_NAME", "META_USER_ATTR_NAME = '" + metadataAttrNameRefMonth + "' AND USER_NAME = '" + groupName + "'", genquery.AS_LIST, callback ) data_size = 0 for row in iter2: data = row[0] temp = json.loads(data) data_size = data_size + int(temp[2]) # no construction for summation required in this case groups.append([groupName, data_size]) return groups
def pep_database_set_avu_metadata_pre(rule_args, callback, rei): # callback.writeLine("serverLog", "pep_database_set_avu_metadata_pre. Arguments: " + str(len(rule_args))) object_name = rule_args[4] object_type = rule_args[3] object_unit = rule_args[7] object_attribute = rule_args[5] # This policy is not using the helper allowAvuChange function as the set operation can also modify units indirectly # Get all AVUs with attribute $id fields = getFieldsForType(callback, object_type, object_name) fields_id = fields['WHERE'] + " AND %s = '$id'" % (fields['a']) rows = genquery.row_iterator([fields['a'], fields['v'], fields['u']], fields_id, genquery.AS_DICT, callback) # From these AVUs extract the unit (root) root_list = [] for row in rows: root_list.append(row[fields['u']]) # Get the unit from the avu that is currently added. for root in root_list: # If the unit start with one of the roots, disallow the operation if str(object_unit).startswith(root + "_"): callback.msiOprDisallowed() # A set operation can also change the unit of existing attributes fields_a = fields['WHERE'] + " AND %s = '%s'" % (fields['a'], object_attribute) rows = genquery.row_iterator([fields['a'], fields['v'], fields['u']], fields_a, genquery.AS_DICT, callback) for row in rows: for root in root_list: # If the unit start with one of the roots, disallow the operation if str(row[fields['u']]).startswith(root + "_"): callback.msiOprDisallowed()
def getCategory(callback, rods_zone, group_name): category = '-1' schemaCategory = 'default' # Find out category based on current group_name. iter = genquery.row_iterator( "META_USER_ATTR_NAME, META_USER_ATTR_VALUE", "USER_GROUP_NAME = '" + group_name + "' AND META_USER_ATTR_NAME like 'category'", genquery.AS_LIST, callback ) for row in iter: category = row[1] if category != '-1': # Test whether found category actually has a collection with XSD's. # If not, fall back to default schema collection. Otherwise use category schema collection # /tempZone/yoda/schemas/default # - metadata.xsd # - vault.xsd xsdCollectionName = '/' + rods_zone + '/yoda/schemas/' + category iter = genquery.row_iterator( "COLL_NAME", "DATA_NAME like '%%.xsd' AND COLL_NAME = '" + xsdCollectionName + "'", genquery.AS_LIST, callback ) for row in iter: schemaCategory = category # As collection is present, the schemaCategory can be assigned the category return schemaCategory
def get_first_eligible_input(callback, input_colln, task_id, sort_key_func=None): METADATA_TAG = Metadata_Tag # Start with a set of all data objects in the input collection # subtract from the set all objects with nonzero task id's set in iRODS metadata eligible_inputs = set( "{COLL_NAME}/{DATA_NAME}".format(**d) for d in \ row_iterator( ["COLL_NAME","DATA_NAME"], "COLL_NAME = '{input_colln}'".format(**locals()), AS_DICT,callback ) )-set( "{COLL_NAME}/{DATA_NAME}".format(**d) for d in \ row_iterator( ["COLL_NAME","DATA_NAME"], "COLL_NAME = '{input_colln}' and META_DATA_ATTR_NAME = '{METADATA_TAG}' " "and META_DATA_ATTR_VALUE != '' ".format(**locals()), AS_DICT,callback ) ) if sort_key_func: eligible_inputs = sorted(list(eligible_inputs), key=sort_key_func) else: eligible_inputs = list(eligible_inputs) chosen_input = None if len(eligible_inputs): chosen_input = eligible_inputs[0] meta_stamp(callback, chosen_input, task_id) return chosen_input
def getUnpreservableFiles(callback, rei, folder, list): zoneName = "" rods_zone = session_vars.get_map(rei)["client_user"]["irods_zone"] # Retrieve JSON list of preservable file formats. json = parseJson(callback, "/" + rods_zone + "/yoda/file_formats/" + list + ".json") preservableFormats = json['formats'] unpreservableFormats = [] # Retrieve all files in collection. iter = genquery.row_iterator("DATA_NAME, COLL_NAME", "COLL_NAME like '%s%%'" % (folder), genquery.AS_LIST, callback) for row in iter: filename, file_extension = os.path.splitext(row[0]) # Convert to lowercase and remove dot. file_extension = (file_extension.lower())[1:] # Check if extention is in preservable format list. if (file_extension not in preservableFormats): unpreservableFormats.append(file_extension) # Remove duplicate file formats. output = [] for x in unpreservableFormats: if x not in output: output.append(x) return {'formats': output}
def iiCheckMetadataXmlForSchemaIdentifier(rule_args, callback, rei): rods_zone = session_vars.get_map(rei)["client_user"]["irods_zone"] callback.writeString("stdout", "[METADATA] Start check for schema identifiers.\n") # Find all research and vault collections, ordered by COLL_ID. iter = genquery.row_iterator( "ORDER(COLL_ID), COLL_NAME", "COLL_NAME like '/%s/home/%%' AND DATA_NAME like 'yoda-metadata%%xml'" % (rods_zone), genquery.AS_LIST, callback ) # Check each collection in batch. for row in iter: coll_id = int(row[0]) coll_name = row[1] pathParts = coll_name.split('/') group_name = pathParts[3] if 'research-' in group_name: checkMetadataXmlForSchemaIdentifier(callback, rods_zone, coll_name, group_name, "yoda-metadata.xml") elif 'vault-' in group_name: # Get vault package path. vault_package = '/'.join(pathParts[:5]) data_name = getLatestVaultMetadataXml(callback, vault_package) if data_name != "": checkMetadataXmlForSchemaIdentifier(callback, rods_zone, vault_package, group_name, data_name) else: callback.writeLine("stdout", "Missing metadata file: %s" % (vault_package)) callback.writeString("stdout", "[METADATA] Finished check for schema identifiers.\n")
def pep_database_del_avu_metadata_pre(rule_args, callback, rei): # callback.writeLine("serverLog", "pep_database_del_avu_metadata_pre. Arguments: " + str(len(rule_args))) object_name = rule_args[5] object_type = rule_args[4] object_attribute = rule_args[6] object_value = rule_args[7] object_unit = rule_args[8] if not allowAvuChange(object_name, object_type, object_unit, callback): callback.msiOprDisallowed() # Wild card removal check if "%" in [object_attribute, object_value, object_unit]: # Get all AVU for the from object fields = getFieldsForType(callback, object_type, object_name) fields_a = fields['WHERE'] # if a,v or u is not wild card add to filter if object_attribute != "%": fields_a = fields_a + " AND %s = '%s'" % (fields['a'], object_attribute) if object_value != "%": fields_a = fields_a + " AND %s = '%s'" % (fields['v'], object_value) if object_unit != "%": fields_a = fields_a + " AND %s = '%s'" % (fields['u'], object_unit) avus = genquery.row_iterator([fields['a'], fields['v'], fields['u']], fields_a, genquery.AS_DICT, callback) for avu in avus: unit = str(avu[fields['u']]) if not allowAvuChange(object_name, object_type, unit, callback): callback.msiOprDisallowed()
def uuRuleGetMonthStoragePerTierForGroup(rule_args, callback, rei): groupName = rule_args[0] currentMonth = int(rule_args[1]) # this is the month that came from the frontend allStorage = [] # list of all month-tier combinations present including their storage size # per month gather month/tier/storage information from metadata: # metadata-attr-name = UUMETADATASTORAGEMONTH + '01'...'12' # metadata-attr-val = [category,tier,storage] ... only tier and storage required wihtin this code for counter in range(0, 11): referenceMonth = currentMonth - counter if referenceMonth < 1: referenceMonth = referenceMonth + 12 metadataAttrNameRefMonth = UUMETADATASTORAGEMONTH + '%0*d' % (2, referenceMonth) iter = genquery.row_iterator( "META_USER_ATTR_VALUE, USER_NAME, USER_GROUP_NAME", "META_USER_ATTR_NAME = '" + metadataAttrNameRefMonth + "' AND USER_NAME = '" + groupName + "'", genquery.AS_LIST, callback ) for row in iter: data = json.loads(row[0]) tierName = data[1] data_size = data[2] # no construction for summation required in this case key = 'month=' + str(referenceMonth) + '-tier=' + tierName allStorage.append({key: data_size}) rule_args[2] = json.dumps(allStorage)
def user_id_for_name(callback, username): user_id="" for i in row_iterator( 'USER_ID,USER_NAME', "USER_NAME = '{}'".format(username), AS_DICT, callback): if 0 == len(user_id): user_id = i['USER_ID'] return user_id
def this_host_tied_to_resc(callback, resc ): import socket this_host = socket.gethostname() tied_host = "" for rescvault in row_iterator( 'RESC_LOC',"RESC_NAME = '{resc}'".format(**locals()), AS_DICT,callback): tied_host = rescvault['RESC_LOC'] return this_host == tied_host
def user_has_access (callback, rei, username, access_type_name, data_object_path='', collection_path=''): access = False access_types = { 'write':'1120', 'read':'1050', 'own':'1200' } user_ID = user_id_for_name (callback,username) do_query = (user_ID != '') if data_object_path and not collection_path : coll_name , data_name = split_irods_path (data_object_path) condition = "DATA_NAME = '{0}' and COLL_NAME = '{1}' "\ "and DATA_ACCESS_USER_ID = '{2}' and DATA_ACCESS_TYPE >= '{3}'".format( data_name, coll_name, user_ID, access_types[access_type_name] ) elif collection_path and not data_object_path: condition = "COLL_NAME = '{0}' and COLL_ACCESS_USER_ID = '{1}' and COLL_ACCESS_TYPE >= '{2}' ".format( collection_path, user_ID, access_types[access_type_name] ) else : do_query = False if do_query: for i in row_iterator( "COLL_NAME", condition, AS_LIST, callback): access = True return access
def getSubcategories(callback, category): categories = set() # Unique subcategories. groupCategories = {} # Group name => { category => .., subcategory => .. } # Collect metadata of each group into `groupCategories` until both # the category and subcategory are available, then add the subcategory # to `categories` if the category name matches. iter = genquery.row_iterator( "USER_GROUP_NAME, META_USER_ATTR_NAME, META_USER_ATTR_VALUE", "USER_TYPE = 'rodsgroup' AND META_USER_ATTR_NAME LIKE '%category'", genquery.AS_LIST, callback ) for row in iter: group = row[0] key = row[1] value = row[2] if group not in groupCategories: groupCategories[group] = {} if key in ['category', 'subcategory']: groupCategories[group][key] = value if ('category' in groupCategories[group] and 'subcategory' in groupCategories[group]): # Metadata complete, now filter on category. if groupCategories[group]['category'] == category: # Bingo, add to the subcategory list. categories.add(groupCategories[group]['subcategory']) del groupCategories[group] return list(categories)
def replicate_data_objects( rule_args , callback , rei): from_object = rule_args[0] to_resource = rule_args[1] from_resource = rule_args[2] trim_after_replication = rule_args[3] if len(to_resource) == 0 or len(from_object) == 0: return path,obj = split_irods_path (from_object) condition = "COLL_NAME = '{0}' and DATA_NAME = '{1}' ".format(path,obj) if from_resource: condition += " and DATA_RESC_NAME = '{}' ".format(from_resource) data_objects = list(row_iterator('DATA_NAME,COLL_NAME,DATA_RESC_NAME,DATA_REPL_NUM', condition, AS_DICT, callback)) if not(data_objects): condition = "COLL_NAME = '{0}' || like '{0}/%' " .format (from_object) if from_resource: condition += " and DATA_RESC_NAME = '{}'".format(from_resource) data_objects = list(row_iterator('DATA_NAME,COLL_NAME,DATA_RESC_NAME,DATA_REPL_NUM', condition, AS_DICT, callback)) replicated = {} for dobj in data_objects: full_path = "{COLL_NAME}/{DATA_NAME}".format(**dobj) if dobj['DATA_RESC_NAME'] == to_resource: replicated[full_path] = True else: old_replication_status = replicated.get(full_path, False) if not old_replication_status: #callback.writeLine("stderr", "replicating: \n" + pprint.pformat(dobj)) retval = callback.msiDataObjRepl( full_path, "destRescName={0}".format(to_resource),0) new_replication_status = retval['status'] replicated [full_path] = new_replication_status if new_replication_status and not(old_replication_status) and trim_after_replication and \ dobj['DATA_RESC_NAME'] == from_resource != "": trim_retval = callback.msiDataObjTrim( "{COLL_NAME}/{DATA_NAME}".format(**dobj), "null", dobj['DATA_REPL_NUM'], "1", "null", 0)
def getUserNameFromUserId(callback, user_id): user_name = "" iter = genquery.row_iterator( "USER_NAME", "USER_ID = '%s'" % (str(user_id)), genquery.AS_LIST, callback ) for row in iter: user_name = row[0] return user_name
def getDataObjSize(callback, coll_name, data_name): data_size = 0 iter = genquery.row_iterator( "DATA_SIZE", "COLL_NAME = '%s' AND DATA_NAME = '%s'" % (coll_name, data_name), genquery.AS_LIST, callback ) for row in iter: data_size = row[0] return data_size
def getCategories(callback): categories = [] iter = genquery.row_iterator( "META_USER_ATTR_VALUE", "USER_TYPE = 'rodsgroup' AND META_USER_ATTR_NAME = 'category'", genquery.AS_LIST, callback ) for row in iter: categories.append(row[0]) return categories
def data_object_physical_path_in_vault(callback, path, resc, force_creation_on_resc, vault_validate = None): #pr = make_logger(callback,'stderr') colln, dataobj = split_irods_path( path ) status = _data_object_exists_targeting_resc( callback, resc, colln, dataobj ) if status != 'already-exists' and force_creation_on_resc.upper() not in ('', 'N','NO','0') : close_rv = {} ; repl_rv = {} ; close_rv = {} if (status == 'need-repl'): repl_rv = callback.msiDataObjRepl(path, "destRescName={}".format(resc), 0) #pr("repl status = %r arg-retn %r " % (repl_rv['status'],repl_rv['arguments'][2])) else: create_rv = callback.msiDataObjCreate(path, "forceFlag=++++destRescName={}".format(resc), 0) descriptor = create_rv['arguments'][2] if type(descriptor) is int and descriptor > 0: close_rv = callback.msiDataObjClose(descriptor,0) #pr("close(%r) status = %r arg-retn %r " % (descriptor,close_rv['status'],close_rv['arguments'][1])) v = {} if type(vault_validate) is dict: # - get vault path to match against data object phys. path v = vault_validate leading_path = "" for rescvault in row_iterator( 'RESC_VAULT_PATH',"RESC_NAME = '{resc}'".format(**locals()), AS_DICT,callback ): leading_path = rescvault['RESC_VAULT_PATH'] phys_path = '' for p in row_iterator("DATA_PATH", "DATA_RESC_NAME = '{resc}' and DATA_NAME = '{dataobj}' and COLL_NAME = '{colln}' ".format(**locals()), AS_DICT,callback): phys_path = p['DATA_PATH'] if leading_path != '' and phys_path.startswith( leading_path ): v['vault_relative_path'] = phys_path [len(leading_path):].lstrip('/') return leading_path
def pep_database_copy_avu_metadata_pre(rule_args, callback, rei): # callback.writeLine("serverLog", "pep_database_copy_avu_metadata_pre. Arguments: " + str(len(rule_args))) object_name_from = rule_args[5] object_type_from = rule_args[3] object_name_to = rule_args[6] object_type_to = rule_args[4] # Get all AVU for the from object fields_from = getFieldsForType(callback, object_type_from, object_name_from) avus_from = genquery.row_iterator([fields_from['a'], fields_from['v'], fields_from['u']], fields_from['WHERE'], genquery.AS_DICT, callback) # Get all AVUs with attribute $id from the to object fields_to = getFieldsForType(callback, object_type_to, object_name_to) fields_id = fields_to['WHERE'] + " AND %s = '$id'" % (fields_to['a']) rows = genquery.row_iterator([fields_to['a'], fields_to['v'], fields_to['u']], fields_id, genquery.AS_DICT, callback) # From these AVUs extract the unit (root) root_list_to = [] for row in rows: root_list_to.append(row[fields_to['u']]) # Regular expression pattern for unit field pattern = re.compile(jsonavu.RE_UNIT) # For all AVUs on the from object check if one starts with the one of the root from the to object for avu in avus_from: for root in root_list_to: # Match unit to extract all info unit = str(avu[fields_from['u']]) # If unit is matching if pattern.match(unit) and unit.startswith(root + "_"): # callback.writeLine("serverLog", "JSON root " + root + " is already in use in the to object") # callback.msiExit("-1101000", "JSON root " + root + " is already in use in the to object") callback.msiOprDisallowed()
def getTierOnResourceName(resourceName, callback): tierName = UUDEFAULTRESOURCETIER # Add default tier as this might not be present in database. # find (possibly present) tier for this resource iter = genquery.row_iterator( "RESC_ID, RESC_NAME, META_RESC_ATTR_NAME, META_RESC_ATTR_VALUE", "RESC_NAME = '" + resourceName + "' AND META_RESC_ATTR_NAME = '" + UURESOURCETIERATTRNAME + "'", genquery.AS_LIST, callback ) for row in iter: tierName = row[3] return tierName
def getProvenanceLog(callback, folder): provenance_log = [] # Retrieve all provenance logs on a folder. iter = genquery.row_iterator( "order(META_COLL_ATTR_VALUE)", "COLL_NAME = '%s' AND META_COLL_ATTR_NAME = 'org_action_log'" % (folder), genquery.AS_LIST, callback) for row in iter: log_item = json.loads(row[0]) provenance_log.append(log_item) return provenance_log
def getCategoriesDatamanager(datamanagerName, callback): categories = [] iter = genquery.row_iterator( "USER_NAME", "USER_TYPE = 'rodsgroup' AND USER_NAME like 'datamanager-%'", genquery.AS_LIST, callback ) for row in iter: # @TODO membership still has to be checked datamanagerGroupname = row[0] temp = datamanagerGroupname.split('-') # 'datamanager-initial' is groupname of datamanager, second part is category categories.append(temp[1]) return categories
def _list_all_application_configs(callback, ctx, save_rows = None, as_lookup=False): # TODO : renaming to irods::compute_to_data rows = scratch = [] if isinstance(save_rows,list): rows = save_rows elif isinstance(save_rows,tuple): rows = list(save_rows) if (rows is scratch) or (rows is save_rows): rows[:] = [ r for r in row_iterator( "COLL_NAME,DATA_NAME,META_DATA_ATTR_VALUE", ("DATA_NAME like '%.json' " "and COLL_NAME like '%/configured_applications' " "and COLL_NAME not like '%/trash/%' " "and META_DATA_ATTR_NAME = 'irods::compute::application' " "and META_DATA_ATTR_VALUE like '_%' " # "and META_DATA_ATTR_UNITS like 'docker' " ).format(**ctx) , AS_DICT, callback) ] return [ "{COLL_NAME}/{DATA_NAME}".format(**row) for row in rows ] if not as_lookup else \ { row["META_DATA_ATTR_VALUE"]:"{COLL_NAME}/{DATA_NAME}".format(**row) for row in rows }
def getLatestVaultMetadataXml(callback, vaultPackage): dataName = "" iter = genquery.row_iterator( "DATA_NAME, DATA_SIZE", "COLL_NAME = '" + vaultPackage + "' AND DATA_NAME like 'yoda-metadata[%].xml'", genquery.AS_LIST, callback ) # Loop through all XMLs. for row in iter: data_name = row[0] data_size = int(row[1]) if dataName == "" or (dataName < data_name and len(dataName) <= len(data_name)): dataName = data_name return dataName
def uuRuleGetResourcesAndTierData(rule_args, callback, rei): resourceList = list() iter = genquery.row_iterator( "RESC_ID, RESC_NAME", "", genquery.AS_LIST, callback ) for row in iter: resourceId = row[0] resourceName = row[1] tierName = getTierOnResourceName(resourceName, callback) resourceList.append({'resourceName': resourceName, 'resourceId': resourceId, 'org_storage_tier': tierName}) rule_args[0] = json.dumps(resourceList)
def pep_api_phy_path_reg_post(rule_args,callback,rei): pr = p(callback) pr( str(rule_args) ) pr("REG ** post ** {!r}".format(rule_args)) # - get file path and resc name for potentially remote netcdf extraction phyPath = "" rescName = "" cndI = rule_args[2].condInput for i in range(cndI.len): if cndI.key[i] == 'filePath': phyPath = cnd.value[i] elif cndI.key[i] == 'destRescName': rescName = cnd.value[i] lgPath = str(rule_args[2].objPath) lgPath_host = "" for x in row_iterator('RESC_LOC' , "RESC_NAME = '{}'".format(rescName), AS_LIST, callback): logPath_host= x[0]
def uuRuleExportMonthlyCategoryStatisticsDM(rule_args, callback, rei): datamanagerUser = rule_args[0] categories = getCategoriesDatamanager(datamanagerUser, callback) allStorage = [] # Select a full year by not limiting UUMETADATASTORAGEMONTH to a perticular month. But only on its presence. # There always is a maximum of one year of history of storage data for category in categories: groupToSubcategory = {} iter = genquery.row_iterator( "META_USER_ATTR_VALUE, META_USER_ATTR_NAME, USER_NAME, USER_GROUP_NAME", "META_USER_ATTR_VALUE like '[\"" + category + "\",%' AND META_USER_ATTR_NAME like '" + UUMETADATASTORAGEMONTH + "%'", genquery.AS_LIST, callback ) for row in iter: attrValue = row[0] month = row[1] month = str(int(month[-2:])) # the month storage data is about, is taken from the attr_name of the AVU groupName = row[3] # Determine subcategory on groupName try: subcategory = groupToSubcategory[groupName] except KeyError: catInfo = groupGetCategoryInfo(groupName, callback) subcategory = catInfo['subcategory'] groupToSubcategory[groupName] = subcategory temp = json.loads(attrValue) category = temp[0] tier = temp[1] storage = int(temp[2]) allStorage.append({'category': category, 'subcategory': subcategory, 'groupname': groupName, 'tier': tier, 'month': month, 'storage': str(storage)}) rule_args[1] = json.dumps(allStorage)
def setJsonSchemaToObj(rule_args, callback, rei): """ This rule stores a given JSON-schema as AVU's to an object :param rule_args: Argument 0: The object name (/nlmumc/P000000003, /nlmumc/projects/metadata.xml, [email protected], demoResc) Argument 1: The object type -d for data object -R for resource -C for collection -u for user Argument 2: URL to the JSON-Schema example https://api.myjson.com/bins/17vejk Argument 3: The JSON root according to https://github.com/MaastrichtUniversity/irods_avu_json. :param callback: :param rei: :return: """ object_name = rule_args[0] object_type = rule_args[1] json_schema_url = rule_args[2] json_root = rule_args[3] # Check if this root has been used before fields = getFieldsForType(callback, object_type, object_name) avus = genquery.row_iterator([fields['a'], fields['v'], fields['u']], fields['WHERE'], genquery.AS_DICT, callback) # Regular expression pattern for unit field pattern = re.compile(jsonavu.RE_UNIT) for avu in avus: # Match unit to extract all info unit = str(avu[fields['u']]) # If unit is matching if pattern.match(unit) and unit.startswith(json_root + "_"): callback.msiExit("-1101000", "JSON root " + json_root + " is already in use") # Delete existing $id AVU for this JSON root callback.msi_rmw_avu(object_type, object_name, '$id', "%", json_root) # Set new $id AVU callback.msi_add_avu(object_type, object_name, '$id', json_schema_url, json_root)
def getMonthlyCategoryStorageStatistics(categories, callback): month = '%0*d' % (2, datetime.now().month) metadataName = UUMETADATASTORAGEMONTH + month storageDict = {} for category in categories: iter = genquery.row_iterator( "META_USER_ATTR_VALUE, META_USER_ATTR_NAME, USER_NAME, USER_GROUP_NAME", "META_USER_ATTR_VALUE like '[\"" + category + "\",%' AND META_USER_ATTR_NAME = '" + metadataName + "'", genquery.AS_LIST, callback ) for row in iter: # hier wordt door alle groepen gezocht, geordend van een category. # per tier moet worden gesommeerd om totale hoeveelheid storage op een tier te verkrijgen. attrValue = row[0] temp = json.loads(attrValue) category = temp[0] tier = temp[1] storage = int(float(temp[2])) try: storageDict[category][tier] = storageDict[category][tier] + storage except KeyError: callback.writeString('serverLog', 'Exception') # if key error, can be either category or category/tier combination is missing try: storageDict[category][tier] = storage except KeyError: storageDict[category] = {tier: storage} # prepare for json output, convert storageDict into dict with keys allStorage = [] for category in storageDict: for tier in storageDict[category]: allStorage.append({'category': category, 'tier': tier, 'storage': str(storageDict[category][tier])}) return json.dumps(allStorage)
def checkDataObjectIntegrity(callback, data_id): # Obtain all replicas of a data object. iter = genquery.row_iterator( "DATA_ID, DATA_NAME, DATA_SIZE, DATA_CHECKSUM, COLL_NAME, RESC_VAULT_PATH, RESC_LOC", "DATA_ID = '%s'" % data_id, genquery.AS_LIST, callback) # Loop through all replicas. for row in iter: data_object = DataObject._make( [row[0], row[1], row[2], row[3], row[4], row[5], row[6]]) # Build file path to data object. coll_name = os.path.join( *(data_object.coll_name.split(os.path.sep)[2:])) file_path = data_object.resc_path + "/" + coll_name + "/" + data_object.name # Check integrity on the resource. remote_rule = "checkDataObjectRemote('%s', '%s', '%s')" % \ (file_path.replace("'", "\\'"), data_object.size, data_object.checksum) callback.remoteExec("%s" % data_object.resc_loc, "", remote_rule, "")
def groupGetCategoryInfo(groupName, callback): category = '' subcategory = '' iter = genquery.row_iterator( "META_USER_ATTR_NAME, META_USER_ATTR_VALUE", "USER_GROUP_NAME = '" + groupName + "' AND META_USER_ATTR_NAME LIKE '%category'", genquery.AS_LIST, callback ) for row in iter: attrName = row[0] attrValue = row[1] if attrName == 'category': category = attrValue elif attrName == 'subcategory': subcategory = attrValue return {'category': category, 'subcategory': subcategory}