def _make_file_path(filenameorid):
try:
fileid = long(filenameorid)
path = '/files/id/%d' % fileid
except ValueError:
path = '/files/name/%s' % escape_url_component(filenameorid)
return path
def _make_file_path(filenameorid):
try:
fileid = long(filenameorid)
path = '/files/id/%d' % fileid
except ValueError:
path = '/files/name/%s' % escape_url_component(filenameorid)
return path
def listFiles(samweb, dimensions=None, defname=None, fileinfo=False, stream=False):
""" list files matching either a dataset definition or a dimensions string
arguments:
dimensions: string (default None)
defname: string definition name (default None)
fileinfo: boolean; if True, return file_id, file_size, event_count
stream: boolean: if True the return value will be a generator and the results will
be progressively returned to the client. Note that this keeps the network connection open until
all the response has been read. (default False)
returns:
a generator producing file names (note that the network connection may not be closed
until you have read the entire list). If fileinfo is true, it will produce
(file_name, file_id, file_size, event_count) tuples
"""
# This can return a potentially long list, so don't preload the result
# instead return a generator which reads it progressively
params = {'format':'plain'}
if fileinfo:
params['fileinfo'] = 1
if defname is not None:
result = samweb.getURL('/definitions/name/%s/files/list' % escape_url_component(defname), params=params,stream=True)
else:
result = samweb._callDimensions('/files/list', dimensions, params, stream=True)
if fileinfo:
output = _make_file_info(result.iter_lines())
else:
output = ifilter( None, (l.strip() for l in result.iter_lines()) )
if stream: return output
else: return list(output)
def getFileLineage(samweb, lineagetype, filenameorid):
""" Return lineage information for a file
arguments:
lineagetype (ie "parents", "children")
name or id of file
"""
result = samweb.getURL(_make_file_path(filenameorid) + '/lineage/' + escape_url_component(lineagetype))
return convert_from_unicode(result.json())
def takeSnapshot(samweb, defname, group=None):
""" Create a snapshot for a existing definition
arguments:
definition name
group: group (default experiment name)
"""
if not group: group = samweb.group
result = samweb.postURL('/definitions/name/%s/snapshot?format=plain' % escape_url_component(defname), {"group":group}, secure=True)
return int(result.text.rstrip())
def deleteDefinition(samweb, defname):
""" Delete a dataset definition
arguments:
definition name
(Definitions that have already been used cannot be deleted)
"""
result = samweb.deleteURL('/definitions/name/%s' % escape_url_component(defname), {}, secure=True)
return result.text.rstrip()
def getFileLineage(samweb, lineagetype, filenameorid):
""" Return lineage information for a file
arguments:
lineagetype (ie "parents", "children")
name or id of file
"""
result = samweb.getURL(
_make_file_path(filenameorid) + '/lineage/' +
escape_url_component(lineagetype))
return convert_from_unicode(result.json())
def listFilesSummary(samweb, dimensions=None, defname=None):
""" return summary of files matching either a dataset definition or a dimensions string
arguments:
dimensions: string (default None)
defname: string definition name (default None)"""
if defname is not None:
result = samweb.getURL('/definitions/name/%s/files/summary' % escape_url_component(defname))
else:
result = samweb._callDimensions('/files/summary', dimensions)
return convert_from_unicode(result.json())
def countFiles(samweb, dimensions=None, defname=None):
""" return count of files matching either a dataset definition or a dimensions string
arguments:
dimensions: string (default None)
defname: string definition name (default None)"""
if defname is not None:
result = samweb.getURL('/definitions/name/%s/files/count' % escape_url_component(defname))
else:
result = samweb._callDimensions('/files/count', dimensions)
return long(result.text.strip())
def deleteDefinition(samweb, defname):
""" Delete a dataset definition
arguments:
definition name
(Definitions that have already been used cannot be deleted)
"""
result = samweb.deleteURL('/definitions/name/%s' %
escape_url_component(defname), {},
secure=True)
return result.text.rstrip()
def takeSnapshot(samweb, defname, group=None):
""" Create a snapshot for a existing definition
arguments:
definition name
group: group (default experiment name)
"""
if not group: group = samweb.group
result = samweb.postURL('/definitions/name/%s/snapshot?format=plain' %
escape_url_component(defname), {"group": group},
secure=True)
return int(result.text.rstrip())
def listFilesSummary(samweb, dimensions=None, defname=None):
""" return summary of files matching either a dataset definition or a dimensions string
arguments:
dimensions: string (default None)
defname: string definition name (default None)"""
if defname is not None:
result = samweb.getURL('/definitions/name/%s/files/summary' %
escape_url_component(defname))
else:
result = samweb._callDimensions('/files/summary', dimensions)
return convert_from_unicode(result.json())
def countFiles(samweb, dimensions=None, defname=None):
""" return count of files matching either a dataset definition or a dimensions string
arguments:
dimensions: string (default None)
defname: string definition name (default None)"""
if defname is not None:
result = samweb.getURL('/definitions/name/%s/files/count' %
escape_url_component(defname))
else:
result = samweb._callDimensions('/files/count', dimensions)
return long(result.text.strip())
def modifyDefinition(samweb, existing_defname, defname=None, description=None):
""" Modify a dataset definition
arguments:
existing definition name
defname: new name for definition
description: new description
"""
params = {}
if defname:
params["defname"] = defname
if description:
params["description"] = description
samweb.putURL('/definitions/name/%s' % escape_url_component(existing_defname), params, secure=True, role='*')
def modifyDefinition(samweb, existing_defname, defname=None, description=None):
""" Modify a dataset definition
arguments:
existing definition name
defname: new name for definition
description: new description
"""
params = {}
if defname:
params["defname"] = defname
if description:
params["description"] = description
samweb.putURL('/definitions/name/%s' %
escape_url_component(existing_defname),
params,
secure=True,
role='*')
def listFiles(samweb,
dimensions=None,
defname=None,
fileinfo=False,
stream=False):
""" list files matching either a dataset definition or a dimensions string
arguments:
dimensions: string (default None)
defname: string definition name (default None)
fileinfo: boolean; if True, return file_id, file_size, event_count
stream: boolean: if True the return value will be a generator and the results will
be progressively returned to the client. Note that this keeps the network connection open until
all the response has been read. (default False)
returns:
a generator producing file names (note that the network connection may not be closed
until you have read the entire list). If fileinfo is true, it will produce
(file_name, file_id, file_size, event_count) tuples
"""
# This can return a potentially long list, so don't preload the result
# instead return a generator which reads it progressively
params = {'format': 'plain'}
if fileinfo:
params['fileinfo'] = 1
if defname is not None:
result = samweb.getURL('/definitions/name/%s/files/list' %
escape_url_component(defname),
params=params,
stream=True)
else:
result = samweb._callDimensions('/files/list',
dimensions,
params,
stream=True)
if fileinfo:
output = _make_file_info(result.iter_lines())
else:
output = ifilter(None, (l.strip() for l in result.iter_lines()))
if stream: return output
else: return list(output)
def _descDefinitionURL(defname):
return '/definitions/name/' + escape_url_component(defname) + '/describe'
def _descDefinitionURL(defname):
return '/definitions/name/' + escape_url_component(defname) + '/describe'
