def __init__(self, config):
TemplatedPage.__init__(self, config)
dbs = config.section_('dbs')
phedex = config.section_('phedex')
dbsconfig = {'dbs':dbs.url, 'dbsinst':dbs.instance,
'dbsparams':dbs.params, 'phedex':phedex.url}
self.dbs = DBS(dbsconfig)
self.securityApi = ""
self.fmConfig = config.section_('fmws')
self.verbose = self.fmConfig.verbose
self.day_transfer = self.fmConfig.day_transfer
self.max_transfer = self.fmConfig.max_transfer
self.file_manager = config.section_('file_manager')
self.transfer_dir = self.file_manager.base_directory
self.download_dir = self.fmConfig.download_area
self.fmgr = FileManager()
self.fmgr.configure(fm_config(config))
self.voms_timer = 0
self.userDict = {}
self.userDictPerDay = {}
self.url = "/filemover"
# prevent users from partial retrieval requests
cherrypy.response.headers['Accept-Ranges'] = 'none'
# internal settings
self.base = '' # defines base path for HREF in templates
self.imgdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'images')
if not os.path.isdir(self.imgdir):
self.imgdir = os.environ['FM_IMAGESPATH']
self.cssdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'css')
if not os.path.isdir(self.cssdir):
self.cssdir = os.environ['FM_CSSPATH']
self.jsdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'js')
if not os.path.isdir(self.jsdir):
self.jsdir = os.environ['FM_JSPATH']
if not os.environ.has_key('YUI_ROOT'):
msg = 'YUI_ROOT is not set'
raise Exception(msg)
self.yuidir = os.environ['YUI_ROOT']
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
self.imgmap = {}
self.yuimap = {}
self.cache = {}
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += ['application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript']
cherryconf.update({'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
'tools.etags.on' : False,
})
class FileMoverService(TemplatedPage):
"""FileMover web-server based on CherryPy"""
def __init__(self, config):
TemplatedPage.__init__(self, config)
dbs = config.section_('dbs')
phedex = config.section_('phedex')
dbsconfig = {'dbs':dbs.url, 'dbsinst':dbs.instance,
'dbsparams':dbs.params, 'phedex':phedex.url}
self.dbs = DBS(dbsconfig)
self.securityApi = ""
self.fmConfig = config.section_('fmws')
self.verbose = self.fmConfig.verbose
self.day_transfer = self.fmConfig.day_transfer
self.max_transfer = self.fmConfig.max_transfer
self.file_manager = config.section_('file_manager')
self.transfer_dir = self.file_manager.base_directory
self.download_dir = self.fmConfig.download_area
self.fmgr = FileManager()
self.fmgr.configure(fm_config(config))
self.voms_timer = 0
self.userDict = {}
self.userDictPerDay = {}
self.url = "/filemover"
# prevent users from partial retrieval requests
cherrypy.response.headers['Accept-Ranges'] = 'none'
# internal settings
self.base = '' # defines base path for HREF in templates
self.imgdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'images')
if not os.path.isdir(self.imgdir):
self.imgdir = os.environ['FM_IMAGESPATH']
self.cssdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'css')
if not os.path.isdir(self.cssdir):
self.cssdir = os.environ['FM_CSSPATH']
self.jsdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'js')
if not os.path.isdir(self.jsdir):
self.jsdir = os.environ['FM_JSPATH']
if not os.environ.has_key('YUI_ROOT'):
msg = 'YUI_ROOT is not set'
raise Exception(msg)
self.yuidir = os.environ['YUI_ROOT']
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
self.imgmap = {}
self.yuimap = {}
self.cache = {}
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += ['application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript']
cherryconf.update({'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
'tools.etags.on' : False,
})
@expose
@tools.secmodv2()
def index(self):
"""default service method"""
page = self.getTopHTML()
user, name = credentials()
self.addUser(user)
page += self.userForm(user, name)
page += self.getBottomHTML()
return page
def addUser(self, user):
"""add user to local cache"""
if not self.userDict.has_key(user):
self.userDict[user] = ([], [])
def makedir(self, user):
"""
Create user dir structure on a server. It has the following structure
<path>
├── download
│ ├── <user>
│ │ └── softlinks
The download/<user> area are used for storage of hardlinks to files in
FileMover pool (for bookkeeping purposes), while softlinks are used to
refer to actual location of the files in a pool.
"""
try:
os.makedirs("%s/%s/softlinks" % (self.download_dir, user))
except Exception as _exc:
pass
def getTopHTML(self):
"""HTML top template"""
page = self.templatepage('templateTop', url=self.url, base=self.base)
return page
def getBottomHTML(self):
"""HTML bottom template"""
page = self.templatepage('templateBottom', version=fm_version)
return page
def updateUserPage(self, user):
"""
Update user HTML page with current status of all LFNs
which belong to that user.
"""
try:
iList, sList = self.userDict[user]
except Exception as _exc:
return ""
page = ""
style = ""
for idx in xrange(0, len(iList)):
lfn = iList[idx]
try:
statusCode, statusMsg = sList[idx]
except Exception as exc:
print "\n### userDict", self.userDict[user]
print_exc(exc)
HTTPError(500, "Server error")
if statusCode == StatusCode.DONE: # append remove request
statusMsg += " | " + removeLfn(lfn)
else: # sanitize msg and append cancel request
msg = cgi.escape(statusMsg) + " | " + cancelLfn(lfn)
if statusMsg.lower().find('error') != -1:
img = ''
else:
img = """<img src="images/loading.gif"/> """
statusMsg = img + msg
if style:
style = ""
else:
style = "zebra"
spanid = spanId(lfn)
page += self.templatepage('templateLfnRow', style=style, lfn=lfn, \
spanid=spanid, statusCode=statusCode, statusMsg=statusMsg)
page += self.templatepage('templateCheckStatus', lfn=lfn)
return page
def updatePageWithLfnInfo(self, user, lfn):
"""Update page with LFN info"""
page = ""
lfnList, statList = self.userDict[user]
self.makedir(user)
if not lfnList:
return ""
try:
idx = lfnList.index(lfn)
statusCode, statusMsg = statList[idx]
if statusCode == StatusCode.DONE:
filename = lfn.split('/')[-1]
pfn = os.path.join(self.transfer_dir, lfn[1:])
if os.path.isfile(pfn):
if not os.path.isfile("%s/%s/%s" \
% (self.download_dir, user, filename)):
try:
os.link(pfn, "%s/%s/%s" \
% (self.download_dir, user, filename))
os.symlink(pfn, "%s/%s/softlinks/%s" \
% (self.download_dir, user, filename))
except Exception as exc:
print_exc(exc)
link = "download/%s/%s" % (user, filename)
filepath = "%s/%s/%s" % (self.download_dir, user, filename)
fileStat = os.stat(filepath)
fileSize = sizeFormat(fileStat[stat.ST_SIZE])
msg = "<a href=\"%s/%s\">Download (%s)</a>" \
% (self.url, link, fileSize)
statList[idx] = (StatusCode.DONE, msg)
else:
statList[idx] = (StatusCode.TRANSFER_STATUS_UNKNOWN, \
StatusMsg.TRANSFER_STATUS_UNKNOWN)
msg = cgi.escape(StatusMsg.TRANSFER_STATUS_UNKNOWN)
page += msg + " | " + removeLfn(lfn)
else:
page += cgi.escape(statusMsg) + " | " + cancelLfn(lfn)
except ValueError as err:
print_exc(err)
print lfn
print self.userDict
except Exception as exc:
print_exc(exc)
print lfn
print self.userDict
return page
def userForm(self, user, name):
"""page forms"""
page = self.templatepage('templateForm', name=name)
page += '<div id="fm_response">'
page += self.checkUserCache(user)
page += '</div>'
return page
@expose
@checkargs
def resolveLfn(self, dataset, run, **_kwargs):
"""look-up LFNs in DBS upon user request"""
lfnList = self.dbs.getFiles(run, dataset)
page = self.templatepage('templateResolveLfns', lfnList=lfnList)
return page
@expose
@checkargs
def reset(self, dn, **_kwargs):
"""Reset user quota for given DN"""
user, name = parse_dn(dn)
self.userDictPerDay[user] = (0, today)
return self.userForm(user, name)
def addLfn(self, user, lfn):
"""add LFN request to the queue"""
# check how may requests in total user placed today
today = time.strftime("%Y%m%d", time.gmtime(time.time()))
if self.userDictPerDay.has_key(user):
nReq, day = self.userDictPerDay[user]
if day == today and nReq > self.day_transfer:
return 0
else:
self.userDictPerDay[user] = (0, today)
else:
self.userDictPerDay[user] = (0, today)
lfnList, statList = self.userDict[user]
# check how may requests user placed at once
if len(lfnList) < int(self.max_transfer):
if not lfnList.count(lfn):
lfnList.append(lfn)
status = (StatusCode.REQUESTED, StatusMsg.REQUESTED)
statList.append(status)
nRequests, day = self.userDictPerDay[user]
self.userDictPerDay[user] = (nRequests+1, today)
else:
return CODES['too_many_lfns'] # 2
else:
return CODES['valid'] # 0
self.userDict[user] = (lfnList, statList)
return CODES['too_many_request'] # 1
def delLfn(self, user, lfn):
"""delete LFN from the queue"""
lfnList, statList = self.userDict[user]
if lfnList.count(lfn):
idx = lfnList.index(lfn)
lfnList.remove(lfn)
statList.remove(statList[idx])
self.userDict[user] = (lfnList, statList)
self._remove(user, lfn)
def checkUserCache(self, user):
"""check users's cache"""
page = ""
lfnList, statList = self.userDict[user]
self.makedir(user)
pfnList = os.listdir("%s/%s/softlinks" % (self.download_dir, user))
for ifile in pfnList:
f = "%s/%s/softlinks/%s" % (self.download_dir, user, ifile)
abspath = os.readlink(f)
if not os.path.isfile(abspath):
# we got orphan link
try:
os.remove(f)
except Exception as _exc:
pass
continue
fileStat = os.stat(abspath)
fileSize = sizeFormat(fileStat[stat.ST_SIZE])
link = "download/%s/%s" % (user, ifile)
lfn = abspath.replace(self.transfer_dir, "")
msg = "<a href=\"%s/%s\">Download (%s)</a> " \
% (self.url, link, fileSize)
if not lfnList.count(lfn):
lfnList.append("%s" % lfn)
status = (StatusCode.DONE, msg)
statList.append(status)
self.userDict[user] = (lfnList, statList)
page += self.updateUserPage(user)
return page
def setStat(self, user, _lfn=None, _status=None):
"""
Update status of LFN in transfer for given user and return
FileManager status code.
"""
statCode = StatusCode.UNKNOWN
self.addUser(user)
lfnList, statList = self.userDict[user]
for idx in xrange(0, len(lfnList)):
lfn = lfnList[idx]
if _lfn and lfn != _lfn:
continue
if _status:
status = _status
else:
status = self.fmgr.status(lfn)
statList[idx] = status
statCode = status[0]
break
self.userDict[user] = (lfnList, statList)
return statCode
def _remove(self, user, lfn):
"""remove requested LFN from the queue"""
ifile = lfn.split("/")[-1]
userdir = "%s/%s" % (self.download_dir, user)
# remove soft-link from user download area
try:
link = "%s/softlinks/%s" % (userdir, ifile)
if os.path.isdir(userdir):
os.unlink(link)
except Exception as _exc:
pass
# remove hard-link from user download area
try:
link = "%s/%s" % (userdir, ifile)
if os.path.isdir(userdir):
os.unlink(link)
except Exception as _exc:
pass
# now time to check if no-one else has a hardlink to pfn,
# if so remove pfn
try:
pfn = self.transfer_dir + lfn
fstat = os.stat(pfn)
if int(fstat[stat.ST_NLINK]) == 1:
# only 1 file exists and no other hard-links to it
os.remove(pfn)
except Exception as _exc:
pass
@expose
@tools.secmodv2()
@checkargs
def remove(self, lfn, **_kwargs):
"""remove requested LFN from the queue"""
user, _ = credentials()
self.delLfn(user, lfn)
try:
self.fmgr.cancel(lfn)
status = StatusCode.REMOVED, StatusMsg.REMOVED
self.setStat(user, lfn, status)
page = 'Removed'
except Exception as exc:
page = handleExc(exc)
return page
def tooManyRequests(self, user):
"""report that user has too many requests"""
page = self.templatepage('templateTooManyRequests', \
max_transfer=self.max_transfer, \
day_transfer=self.day_transfer, fulldesc=False)
if self.userDictPerDay.has_key(user):
today = time.strftime("%Y%m%d", time.gmtime(time.time()))
nReq, day = self.userDictPerDay[user]
if day != today and nReq > self.day_transfer:
page = self.templatepage('templateTooManyRequests', \
max_transfer=self.max_transfer, \
day_transfer=self.day_transfer, fulldesc=True)
page += self.updateUserPage(user)
return page
@expose
@tools.secmodv2()
@checkargs
def request(self, lfn, **kwargs):
"""place LFN request"""
user, name = credentials()
html = kwargs.get('external', 0)
self.addUser(user)
lfn = lfn.strip()
lfnStatus = self.addLfn(user, lfn)
if not lfnStatus:
return self.tooManyRequests(user)
page = ""
try:
if lfnStatus == 1:
self.fmgr.request(lfn)
page += 'Requested'
else:
page += 'Already in queue'
page += self.updateUserPage(user)
except Exception as exc:
page = handleExc(exc)
if html:
main = self.getTopHTML()
main += self.templatepage('templateForm', name=name)
main += '<div id="fm_response">'
page += self.templatepage('templateCheckStatus', lfn=lfn)
main += page
main += '</div>'
main += self.getBottomHTML()
return main
return page
@expose
@tools.secmodv2()
@checkargs
def cancel(self, lfn, **_kwargs):
"""cancel LFN request"""
user, _ = credentials()
self.delLfn(user, lfn)
page = ""
try:
self.fmgr.cancel(lfn)
status = StatusCode.CANCELLED, StatusMsg.CANCELLED
self.setStat(user, lfn, status)
page = 'Request cancelled'
except Exception as exc:
page = handleExc(exc)
# page += self.updateUserPage(user)
return page
@expose
@tools.secmodv2()
@checkargs
def statusOne(self, lfn, **_kwargs):
"""return status of requested LFN"""
cherrypy.response.headers['Cache-control'] = 'no-cache'
cherrypy.response.headers['Expire'] = 0
user, _ = credentials()
page = ""
lfn = lfn.strip()
spanid = spanId(lfn)
page += """<span id="%s" name="%s">""" % (spanid, spanid)
statCode = 0
stop_codes = [StatusCode.TRANSFER_FAILED, StatusCode.CANCELLED,
StatusCode.REMOVED]
try:
statCode = self.setStat(user, lfn)
if statCode == StatusCode.FAILED:
# this happen when proxy is expired, need to look at a log
page += "Request fails. "
elif statCode == StatusCode.UNKNOWN:
page += 'lfn status unknown. '
elif statCode == StatusCode.CANCELLED:
page += 'Transfer is cancelled. '
elif statCode == StatusCode.REMOVED:
page += 'lfn is removed. '
elif statCode and statCode not in stop_codes:
page += self.templatepage('templateLoading', msg="")
page += self.updatePageWithLfnInfo(user, lfn)
except Exception as exc:
page += handleExc(exc)
page += "</span>"
return page
@expose
def download(self, *args, **_kwargs):
"""Server FileMover download area"""
ctype = 'application/octet-stream'
path = '%s/%s/%s' % (self.download_dir, args[0], args[1])
if os.path.isfile(path):
return serve_file(path, content_type=ctype)
raise HTTPError(500, 'File not found')
@expose
def images(self, *args, **_kwargs):
"""
Serve static images.
"""
args = list(args)
_scripts = check_scripts(args, self.imgmap, self.imgdir)
mime_types = ['*/*', 'image/gif', 'image/png',
'image/jpg', 'image/jpeg']
accepts = cherrypy.request.headers.elements('Accept')
for accept in accepts:
if accept.value in mime_types and len(args) == 1 \
and self.imgmap.has_key(args[0]):
image = self.imgmap[args[0]]
# use image extension to pass correct content type
ctype = 'image/%s' % image.split('.')[-1]
cherrypy.response.headers['Content-type'] = ctype
if os.path.isfile(image):
return serve_file(image, content_type=ctype)
raise HTTPError(500, 'Image file not found')
@expose
@tools.gzip()
def yui(self, *args, **_kwargs):
"""
Serve YUI library. YUI files has disperse directory structure, so
input args can be in a form of (build, container, container.js)
which corresponds to a single YUI JS file
build/container/container.js
"""
cherrypy.response.headers['Content-Type'] = \
["text/css", "application/javascript"]
args = ['/'.join(args)] # preserve YUI dir structure
scripts = check_scripts(args, self.yuimap, self.yuidir)
return self.serve_files(args, scripts, self.yuimap)
@expose
@tools.gzip()
def css(self, *args, **_kwargs):
"""
Cat together the specified css files and return a single css include.
Multiple files can be supplied in a form of file1&file2&file3
"""
cherrypy.response.headers['Content-Type'] = "text/css"
args = parse_args(args)
scripts = check_scripts(args, self.cssmap, self.cssdir)
return self.serve_files(args, scripts, self.cssmap, 'css', True)
@expose
@tools.gzip()
def js(self, *args, **_kwargs):
"""
Cat together the specified js files and return a single js include.
Multiple files can be supplied in a form of file1&file2&file3
"""
cherrypy.response.headers['Content-Type'] = "application/javascript"
args = parse_args(args)
scripts = check_scripts(args, self.jsmap, self.jsdir)
return self.serve_files(args, scripts, self.jsmap)
def serve_files(self, args, scripts, _map, datatype='', minimize=False):
"""
Return asked set of files for JS, YUI, CSS.
"""
idx = "-".join(scripts)
if idx not in self.cache.keys():
data = ''
if datatype == 'css':
data = '@CHARSET "UTF-8";'
for script in args:
path = os.path.join(sys.path[0], _map[script])
path = os.path.normpath(path)
ifile = open(path)
data = "\n".join ([data, ifile.read().\
replace('@CHARSET "UTF-8";', '')])
ifile.close()
if datatype == 'css':
set_headers("text/css")
if minimize:
self.cache[idx] = minify(data)
else:
self.cache[idx] = data
return self.cache[idx]
