def _make_file_path(filenameorid): try: fileid = long(filenameorid) path = '/files/id/%d' % fileid except ValueError: path = '/files/name/%s' % escape_url_component(filenameorid) return path
def listFiles(samweb, dimensions=None, defname=None, fileinfo=False, stream=False): """ list files matching either a dataset definition or a dimensions string arguments: dimensions: string (default None) defname: string definition name (default None) fileinfo: boolean; if True, return file_id, file_size, event_count stream: boolean: if True the return value will be a generator and the results will be progressively returned to the client. Note that this keeps the network connection open until all the response has been read. (default False) returns: a generator producing file names (note that the network connection may not be closed until you have read the entire list). If fileinfo is true, it will produce (file_name, file_id, file_size, event_count) tuples """ # This can return a potentially long list, so don't preload the result # instead return a generator which reads it progressively params = {'format':'plain'} if fileinfo: params['fileinfo'] = 1 if defname is not None: result = samweb.getURL('/definitions/name/%s/files/list' % escape_url_component(defname), params=params,stream=True) else: result = samweb._callDimensions('/files/list', dimensions, params, stream=True) if fileinfo: output = _make_file_info(result.iter_lines()) else: output = ifilter( None, (l.strip() for l in result.iter_lines()) ) if stream: return output else: return list(output)
def getFileLineage(samweb, lineagetype, filenameorid): """ Return lineage information for a file arguments: lineagetype (ie "parents", "children") name or id of file """ result = samweb.getURL(_make_file_path(filenameorid) + '/lineage/' + escape_url_component(lineagetype)) return convert_from_unicode(result.json())
def takeSnapshot(samweb, defname, group=None): """ Create a snapshot for a existing definition arguments: definition name group: group (default experiment name) """ if not group: group = samweb.group result = samweb.postURL('/definitions/name/%s/snapshot?format=plain' % escape_url_component(defname), {"group":group}, secure=True) return int(result.text.rstrip())
def deleteDefinition(samweb, defname): """ Delete a dataset definition arguments: definition name (Definitions that have already been used cannot be deleted) """ result = samweb.deleteURL('/definitions/name/%s' % escape_url_component(defname), {}, secure=True) return result.text.rstrip()
def getFileLineage(samweb, lineagetype, filenameorid): """ Return lineage information for a file arguments: lineagetype (ie "parents", "children") name or id of file """ result = samweb.getURL( _make_file_path(filenameorid) + '/lineage/' + escape_url_component(lineagetype)) return convert_from_unicode(result.json())
def listFilesSummary(samweb, dimensions=None, defname=None): """ return summary of files matching either a dataset definition or a dimensions string arguments: dimensions: string (default None) defname: string definition name (default None)""" if defname is not None: result = samweb.getURL('/definitions/name/%s/files/summary' % escape_url_component(defname)) else: result = samweb._callDimensions('/files/summary', dimensions) return convert_from_unicode(result.json())
def countFiles(samweb, dimensions=None, defname=None): """ return count of files matching either a dataset definition or a dimensions string arguments: dimensions: string (default None) defname: string definition name (default None)""" if defname is not None: result = samweb.getURL('/definitions/name/%s/files/count' % escape_url_component(defname)) else: result = samweb._callDimensions('/files/count', dimensions) return long(result.text.strip())
def takeSnapshot(samweb, defname, group=None): """ Create a snapshot for a existing definition arguments: definition name group: group (default experiment name) """ if not group: group = samweb.group result = samweb.postURL('/definitions/name/%s/snapshot?format=plain' % escape_url_component(defname), {"group": group}, secure=True) return int(result.text.rstrip())
def modifyDefinition(samweb, existing_defname, defname=None, description=None): """ Modify a dataset definition arguments: existing definition name defname: new name for definition description: new description """ params = {} if defname: params["defname"] = defname if description: params["description"] = description samweb.putURL('/definitions/name/%s' % escape_url_component(existing_defname), params, secure=True, role='*')
def listFiles(samweb, dimensions=None, defname=None, fileinfo=False, stream=False): """ list files matching either a dataset definition or a dimensions string arguments: dimensions: string (default None) defname: string definition name (default None) fileinfo: boolean; if True, return file_id, file_size, event_count stream: boolean: if True the return value will be a generator and the results will be progressively returned to the client. Note that this keeps the network connection open until all the response has been read. (default False) returns: a generator producing file names (note that the network connection may not be closed until you have read the entire list). If fileinfo is true, it will produce (file_name, file_id, file_size, event_count) tuples """ # This can return a potentially long list, so don't preload the result # instead return a generator which reads it progressively params = {'format': 'plain'} if fileinfo: params['fileinfo'] = 1 if defname is not None: result = samweb.getURL('/definitions/name/%s/files/list' % escape_url_component(defname), params=params, stream=True) else: result = samweb._callDimensions('/files/list', dimensions, params, stream=True) if fileinfo: output = _make_file_info(result.iter_lines()) else: output = ifilter(None, (l.strip() for l in result.iter_lines())) if stream: return output else: return list(output)
def _descDefinitionURL(defname): return '/definitions/name/' + escape_url_component(defname) + '/describe'