def __init__(self, config): TemplatedPage.__init__(self, config) dbs = config.section_('dbs') phedex = config.section_('phedex') dbsconfig = {'dbs':dbs.url, 'dbsinst':dbs.instance, 'dbsparams':dbs.params, 'phedex':phedex.url} self.dbs = DBS(dbsconfig) self.securityApi = "" self.fmConfig = config.section_('fmws') self.verbose = self.fmConfig.verbose self.day_transfer = self.fmConfig.day_transfer self.max_transfer = self.fmConfig.max_transfer self.file_manager = config.section_('file_manager') self.transfer_dir = self.file_manager.base_directory self.download_dir = self.fmConfig.download_area self.fmgr = FileManager() self.fmgr.configure(fm_config(config)) self.voms_timer = 0 self.userDict = {} self.userDictPerDay = {} self.url = "/filemover" # prevent users from partial retrieval requests cherrypy.response.headers['Accept-Ranges'] = 'none' # internal settings self.base = '' # defines base path for HREF in templates self.imgdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'images') if not os.path.isdir(self.imgdir): self.imgdir = os.environ['FM_IMAGESPATH'] self.cssdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'css') if not os.path.isdir(self.cssdir): self.cssdir = os.environ['FM_CSSPATH'] self.jsdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'js') if not os.path.isdir(self.jsdir): self.jsdir = os.environ['FM_JSPATH'] if not os.environ.has_key('YUI_ROOT'): msg = 'YUI_ROOT is not set' raise Exception(msg) self.yuidir = os.environ['YUI_ROOT'] # To be filled at run time self.cssmap = {} self.jsmap = {} self.imgmap = {} self.yuimap = {} self.cache = {} # Update CherryPy configuration mime_types = ['text/css'] mime_types += ['application/javascript', 'text/javascript', 'application/x-javascript', 'text/x-javascript'] cherryconf.update({'tools.encode.on': True, 'tools.gzip.on': True, 'tools.gzip.mime_types': mime_types, 'tools.etags.on' : False, })
def __init__(self, cp): super(FileLookup, self).__init__(cp) self.cp = cp self.section = "file_lookup" self.priorities = self._parse_priority_rules() dbsurl = cp.get('dbs', 'url') dbsinst = cp.get('dbs', 'instance') dbsparams = cp.get('dbs', 'params') self.phedex_url = cp.get('phedex', 'url') dbsconfig = {'dbs':dbsurl, 'dbsinst':dbsinst, 'dbsparams':dbsparams, 'phedex':self.phedex_url} self._dbs = DBS(dbsconfig) self.sitedb_url = cp.get('sitedb', 'url') self.sitedb = SiteDBManager(self.sitedb_url) self._downSites = [] self._lastSiteQuery = 0 self._lock = threading.Lock() self._lfns = {} self._lfns_cache = {} self.acquireTURL = self.acquireValue self.releaseTURL = self.releaseKey
class FileLookup(MappingManager): """Main class which perform LFN/PFN/Site/SE operations""" def __init__(self, cp): super(FileLookup, self).__init__(cp) self.cp = cp self.section = "file_lookup" self.priorities = self._parse_priority_rules() dbsurl = cp.get('dbs', 'url') dbsinst = cp.get('dbs', 'instance') dbsparams = cp.get('dbs', 'params') self.phedex_url = cp.get('phedex', 'url') dbsconfig = {'dbs':dbsurl, 'dbsinst':dbsinst, 'dbsparams':dbsparams, 'phedex':self.phedex_url} self._dbs = DBS(dbsconfig) self.sitedb_url = cp.get('sitedb', 'url') self.sitedb = SiteDBManager(self.sitedb_url) self._downSites = [] self._lastSiteQuery = 0 self._lock = threading.Lock() self._lfns = {} self._lfns_cache = {} self.acquireTURL = self.acquireValue self.releaseTURL = self.releaseKey def acquireTURL(self, SURL): """Get TURL for provided SURL""" self.log.info("Looking up TURL for SURL %s." % SURL) TURL = self.acquireValue(SURL) self.log.info("Found TURL %s for SURL %s." % (TURL, SURL)) return TURL def releaseTURL(self, SURL): """Release TURL for provided SURL""" self.log.info("Releasing SURL %s." % SURL) self.releaseTURL(SURL) def replicas(self, lfn, token=None, user=None): """Find LFN replicas in PhEDEx data-service""" self.log.info("Looking for the block of LFN %s." % lfn) block = self._dbs.blockLookup(lfn) query = {'block':block} self.log.info("Looking for replicas of %s" % block) results = phedex_datasvc('fileReplicas', self.phedex_url, block=block) blocks = [i for i in results['phedex']['block'] \ if i.get('name', None) == block] if not blocks: raise Exception("Requested LFN does not exist in any block known " \ "to PhEDEx.") block = blocks[0] files = [i for i in block.get('file', []) \ if i.get('name', None) == lfn] if not files: raise Exception("Internal error: PhEDEx does not think LFN is in " \ "the same block as DBS does.") file = files[0] replicas = [i['node'] for i in file.get('replica', []) if 'node' in i] self.log.info("There are the following replicas of %s: %s." % \ (lfn, ', '.join(replicas))) return replicas def _parse_priority_rules(self): """Parse priority rules""" priority_dict = {} name_regexp = re.compile('priority_([0-9]+)') try: rules = self.cp.items(self.section) except: rules = {} if not rules: raise Exception("FileMover configured withot priority rules") for name, value in rules: value = value.strip() value = re.compile(value) m = name_regexp.match(name) if not m: continue priority = long(m.groups()[0]) priority_dict[priority] = value return priority_dict def _getSiteStatus(self): """ Update the list of down/bad sites """ if time.time() - self._lastSiteQuery > 600: # Update the list of bad sites. self._lastSiteQuery = time.time() def removeBadSites(self, sites): """ Given a list of sites, remove any which we do not want to transfer with for some reason. """ self._getSiteStatus() filtered_list = [] for site in sites: if site not in self._downSites: filtered_list.append(site) return filtered_list def pickSite(self, replicas, exclude_list=None): """Pick up site from provided replicases and exclude site list""" priorities = self.priorities.keys() priorities.sort() source = None for priority in priorities: for site in replicas: if exclude_list and exclude_list.count(site): continue m = self.priorities[priority].search(site) if m: source = site break if source != None: break if source == None: raise ValueError("Could not match site to any priority. " \ "Possible sources: %s" % str(replicas)) return source def mapLFN(self, site, lfn, protocol=None): """Map LFN to given site""" if not protocol: protocol = 'srmv2' data = {'lfn': lfn, 'node': site} if protocol: data['protocol'] = protocol self.log.info("Mapping LFN %s for site %s using PhEDEx datasvc." % \ (lfn, site)) data = phedex_datasvc('lfn2pfn', self.phedex_url, **data) try: pfn = data['phedex']['mapping'][0]['pfn'] except: raise Exception("PhEDEx data service did not return a PFN!") self.log.info("PhEDEx data service returned PFN %s for LFN %s." % \ (pfn, lfn)) return pfn def getPFN(self, lfn, protocol=None, exclude_sites=None): """Get PFN for given LFN""" # replicas = self.replicas(lfn) # if len(replicas) == 0: # raise Exception("The LFN %s has no known replicas in PhEDEx.") # site = self.pickSite(replicas) # pfn = self.mapLFN(site, lfn) # return pfn self._lock.acquire() site = '' try: key = (lfn, protocol) if key in self._lfns and time.time() - \ self._lfns_cache.get(key, 0) < 10: return self._lfns[key], site finally: self._lock.release() try: replicas = self.replicas(lfn) if len(replicas) == 0: raise \ Exception("The LFN %s has no known replicas in PhEDEx."%lfn) site = self.pickSite(replicas, exclude_sites) except: bsList = self._dbs.blockSiteLookup(lfn) seList = [s for b, s in bsList] msg = "Fail to look-up T[1-3] CMS site for\n" msg += "LFN=%s\nSE's list %s\n" % (lfn, seList) if not seList: raise Exception(msg) site = self.getSiteFromSDB(seList, exclude_sites) if not site: raise Exception(msg) pfn = self.mapLFN(site, lfn, protocol=protocol) self._lock.acquire() try: key = (lfn, protocol) self._lfns[key] = pfn self._lfns_cache[key] = time.time() finally: self._lock.release() return pfn, site def getSiteFromSDB(self, seList, exclude_sites): """ Get SE names for give cms names """ sites = [] for sename in seList: site = self.sitedb.get_name(sename) if site: sites.append(site) site = self.pickSite(sites, exclude_sites) return site def _lookup(self, SURL): """ Return the corresponding gsiftp TURL for a given SURL. """ # SURL (Storage URL, aka PFN) should be in a form # <sfn|srm>://<SE_hostname>/<some_string>.root pat = re.compile("(sfn|srm)://[a-zA-Z0-9].*.*root$") if pat.match(SURL): raise Exception("Bad SURL: %s" % SURL) options = "-T srmv2 -b -p gsiftp" cmd = "lcg-getturls %s %s" % (options, SURL) self.log.info("Looking up TURL for %s." % SURL) print cmd fd = os.popen(cmd) turl = fd.read() print turl if fd.close(): if not turl.startswith("gsiftp://"): # Sometimes lcg-* segfaults self.log.error("Unable to get TURL for SURL %s." % SURL) self.log.error("Error message: %s" % turl) raise ValueError("Unable to get TURL for SURL %s." % SURL) turl = turl.strip() self.log.info("Found TURL %s for %s." % (turl, SURL)) return turl def _release(self, SURL): """
class FileMoverService(TemplatedPage): """FileMover web-server based on CherryPy""" def __init__(self, config): TemplatedPage.__init__(self, config) dbs = config.section_('dbs') phedex = config.section_('phedex') dbsconfig = {'dbs':dbs.url, 'dbsinst':dbs.instance, 'dbsparams':dbs.params, 'phedex':phedex.url} self.dbs = DBS(dbsconfig) self.securityApi = "" self.fmConfig = config.section_('fmws') self.verbose = self.fmConfig.verbose self.day_transfer = self.fmConfig.day_transfer self.max_transfer = self.fmConfig.max_transfer self.file_manager = config.section_('file_manager') self.transfer_dir = self.file_manager.base_directory self.download_dir = self.fmConfig.download_area self.fmgr = FileManager() self.fmgr.configure(fm_config(config)) self.voms_timer = 0 self.userDict = {} self.userDictPerDay = {} self.url = "/filemover" # prevent users from partial retrieval requests cherrypy.response.headers['Accept-Ranges'] = 'none' # internal settings self.base = '' # defines base path for HREF in templates self.imgdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'images') if not os.path.isdir(self.imgdir): self.imgdir = os.environ['FM_IMAGESPATH'] self.cssdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'css') if not os.path.isdir(self.cssdir): self.cssdir = os.environ['FM_CSSPATH'] self.jsdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'js') if not os.path.isdir(self.jsdir): self.jsdir = os.environ['FM_JSPATH'] if not os.environ.has_key('YUI_ROOT'): msg = 'YUI_ROOT is not set' raise Exception(msg) self.yuidir = os.environ['YUI_ROOT'] # To be filled at run time self.cssmap = {} self.jsmap = {} self.imgmap = {} self.yuimap = {} self.cache = {} # Update CherryPy configuration mime_types = ['text/css'] mime_types += ['application/javascript', 'text/javascript', 'application/x-javascript', 'text/x-javascript'] cherryconf.update({'tools.encode.on': True, 'tools.gzip.on': True, 'tools.gzip.mime_types': mime_types, 'tools.etags.on' : False, }) @expose @tools.secmodv2() def index(self): """default service method""" page = self.getTopHTML() user, name = credentials() self.addUser(user) page += self.userForm(user, name) page += self.getBottomHTML() return page def addUser(self, user): """add user to local cache""" if not self.userDict.has_key(user): self.userDict[user] = ([], []) def makedir(self, user): """ Create user dir structure on a server. It has the following structure <path> ├── download │ ├── <user> │ │ └── softlinks The download/<user> area are used for storage of hardlinks to files in FileMover pool (for bookkeeping purposes), while softlinks are used to refer to actual location of the files in a pool. """ try: os.makedirs("%s/%s/softlinks" % (self.download_dir, user)) except Exception as _exc: pass def getTopHTML(self): """HTML top template""" page = self.templatepage('templateTop', url=self.url, base=self.base) return page def getBottomHTML(self): """HTML bottom template""" page = self.templatepage('templateBottom', version=fm_version) return page def updateUserPage(self, user): """ Update user HTML page with current status of all LFNs which belong to that user. """ try: iList, sList = self.userDict[user] except Exception as _exc: return "" page = "" style = "" for idx in xrange(0, len(iList)): lfn = iList[idx] try: statusCode, statusMsg = sList[idx] except Exception as exc: print "\n### userDict", self.userDict[user] print_exc(exc) HTTPError(500, "Server error") if statusCode == StatusCode.DONE: # append remove request statusMsg += " | " + removeLfn(lfn) else: # sanitize msg and append cancel request msg = cgi.escape(statusMsg) + " | " + cancelLfn(lfn) if statusMsg.lower().find('error') != -1: img = '' else: img = """<img src="images/loading.gif"/> """ statusMsg = img + msg if style: style = "" else: style = "zebra" spanid = spanId(lfn) page += self.templatepage('templateLfnRow', style=style, lfn=lfn, \ spanid=spanid, statusCode=statusCode, statusMsg=statusMsg) page += self.templatepage('templateCheckStatus', lfn=lfn) return page def updatePageWithLfnInfo(self, user, lfn): """Update page with LFN info""" page = "" lfnList, statList = self.userDict[user] self.makedir(user) if not lfnList: return "" try: idx = lfnList.index(lfn) statusCode, statusMsg = statList[idx] if statusCode == StatusCode.DONE: filename = lfn.split('/')[-1] pfn = os.path.join(self.transfer_dir, lfn[1:]) if os.path.isfile(pfn): if not os.path.isfile("%s/%s/%s" \ % (self.download_dir, user, filename)): try: os.link(pfn, "%s/%s/%s" \ % (self.download_dir, user, filename)) os.symlink(pfn, "%s/%s/softlinks/%s" \ % (self.download_dir, user, filename)) except Exception as exc: print_exc(exc) link = "download/%s/%s" % (user, filename) filepath = "%s/%s/%s" % (self.download_dir, user, filename) fileStat = os.stat(filepath) fileSize = sizeFormat(fileStat[stat.ST_SIZE]) msg = "<a href=\"%s/%s\">Download (%s)</a>" \ % (self.url, link, fileSize) statList[idx] = (StatusCode.DONE, msg) else: statList[idx] = (StatusCode.TRANSFER_STATUS_UNKNOWN, \ StatusMsg.TRANSFER_STATUS_UNKNOWN) msg = cgi.escape(StatusMsg.TRANSFER_STATUS_UNKNOWN) page += msg + " | " + removeLfn(lfn) else: page += cgi.escape(statusMsg) + " | " + cancelLfn(lfn) except ValueError as err: print_exc(err) print lfn print self.userDict except Exception as exc: print_exc(exc) print lfn print self.userDict return page def userForm(self, user, name): """page forms""" page = self.templatepage('templateForm', name=name) page += '<div id="fm_response">' page += self.checkUserCache(user) page += '</div>' return page @expose @checkargs def resolveLfn(self, dataset, run, **_kwargs): """look-up LFNs in DBS upon user request""" lfnList = self.dbs.getFiles(run, dataset) page = self.templatepage('templateResolveLfns', lfnList=lfnList) return page @expose @checkargs def reset(self, dn, **_kwargs): """Reset user quota for given DN""" user, name = parse_dn(dn) self.userDictPerDay[user] = (0, today) return self.userForm(user, name) def addLfn(self, user, lfn): """add LFN request to the queue""" # check how may requests in total user placed today today = time.strftime("%Y%m%d", time.gmtime(time.time())) if self.userDictPerDay.has_key(user): nReq, day = self.userDictPerDay[user] if day == today and nReq > self.day_transfer: return 0 else: self.userDictPerDay[user] = (0, today) else: self.userDictPerDay[user] = (0, today) lfnList, statList = self.userDict[user] # check how may requests user placed at once if len(lfnList) < int(self.max_transfer): if not lfnList.count(lfn): lfnList.append(lfn) status = (StatusCode.REQUESTED, StatusMsg.REQUESTED) statList.append(status) nRequests, day = self.userDictPerDay[user] self.userDictPerDay[user] = (nRequests+1, today) else: return CODES['too_many_lfns'] # 2 else: return CODES['valid'] # 0 self.userDict[user] = (lfnList, statList) return CODES['too_many_request'] # 1 def delLfn(self, user, lfn): """delete LFN from the queue""" lfnList, statList = self.userDict[user] if lfnList.count(lfn): idx = lfnList.index(lfn) lfnList.remove(lfn) statList.remove(statList[idx]) self.userDict[user] = (lfnList, statList) self._remove(user, lfn) def checkUserCache(self, user): """check users's cache""" page = "" lfnList, statList = self.userDict[user] self.makedir(user) pfnList = os.listdir("%s/%s/softlinks" % (self.download_dir, user)) for ifile in pfnList: f = "%s/%s/softlinks/%s" % (self.download_dir, user, ifile) abspath = os.readlink(f) if not os.path.isfile(abspath): # we got orphan link try: os.remove(f) except Exception as _exc: pass continue fileStat = os.stat(abspath) fileSize = sizeFormat(fileStat[stat.ST_SIZE]) link = "download/%s/%s" % (user, ifile) lfn = abspath.replace(self.transfer_dir, "") msg = "<a href=\"%s/%s\">Download (%s)</a> " \ % (self.url, link, fileSize) if not lfnList.count(lfn): lfnList.append("%s" % lfn) status = (StatusCode.DONE, msg) statList.append(status) self.userDict[user] = (lfnList, statList) page += self.updateUserPage(user) return page def setStat(self, user, _lfn=None, _status=None): """ Update status of LFN in transfer for given user and return FileManager status code. """ statCode = StatusCode.UNKNOWN self.addUser(user) lfnList, statList = self.userDict[user] for idx in xrange(0, len(lfnList)): lfn = lfnList[idx] if _lfn and lfn != _lfn: continue if _status: status = _status else: status = self.fmgr.status(lfn) statList[idx] = status statCode = status[0] break self.userDict[user] = (lfnList, statList) return statCode def _remove(self, user, lfn): """remove requested LFN from the queue""" ifile = lfn.split("/")[-1] userdir = "%s/%s" % (self.download_dir, user) # remove soft-link from user download area try: link = "%s/softlinks/%s" % (userdir, ifile) if os.path.isdir(userdir): os.unlink(link) except Exception as _exc: pass # remove hard-link from user download area try: link = "%s/%s" % (userdir, ifile) if os.path.isdir(userdir): os.unlink(link) except Exception as _exc: pass # now time to check if no-one else has a hardlink to pfn, # if so remove pfn try: pfn = self.transfer_dir + lfn fstat = os.stat(pfn) if int(fstat[stat.ST_NLINK]) == 1: # only 1 file exists and no other hard-links to it os.remove(pfn) except Exception as _exc: pass @expose @tools.secmodv2() @checkargs def remove(self, lfn, **_kwargs): """remove requested LFN from the queue""" user, _ = credentials() self.delLfn(user, lfn) try: self.fmgr.cancel(lfn) status = StatusCode.REMOVED, StatusMsg.REMOVED self.setStat(user, lfn, status) page = 'Removed' except Exception as exc: page = handleExc(exc) return page def tooManyRequests(self, user): """report that user has too many requests""" page = self.templatepage('templateTooManyRequests', \ max_transfer=self.max_transfer, \ day_transfer=self.day_transfer, fulldesc=False) if self.userDictPerDay.has_key(user): today = time.strftime("%Y%m%d", time.gmtime(time.time())) nReq, day = self.userDictPerDay[user] if day != today and nReq > self.day_transfer: page = self.templatepage('templateTooManyRequests', \ max_transfer=self.max_transfer, \ day_transfer=self.day_transfer, fulldesc=True) page += self.updateUserPage(user) return page @expose @tools.secmodv2() @checkargs def request(self, lfn, **kwargs): """place LFN request""" user, name = credentials() html = kwargs.get('external', 0) self.addUser(user) lfn = lfn.strip() lfnStatus = self.addLfn(user, lfn) if not lfnStatus: return self.tooManyRequests(user) page = "" try: if lfnStatus == 1: self.fmgr.request(lfn) page += 'Requested' else: page += 'Already in queue' page += self.updateUserPage(user) except Exception as exc: page = handleExc(exc) if html: main = self.getTopHTML() main += self.templatepage('templateForm', name=name) main += '<div id="fm_response">' page += self.templatepage('templateCheckStatus', lfn=lfn) main += page main += '</div>' main += self.getBottomHTML() return main return page @expose @tools.secmodv2() @checkargs def cancel(self, lfn, **_kwargs): """cancel LFN request""" user, _ = credentials() self.delLfn(user, lfn) page = "" try: self.fmgr.cancel(lfn) status = StatusCode.CANCELLED, StatusMsg.CANCELLED self.setStat(user, lfn, status) page = 'Request cancelled' except Exception as exc: page = handleExc(exc) # page += self.updateUserPage(user) return page @expose @tools.secmodv2() @checkargs def statusOne(self, lfn, **_kwargs): """return status of requested LFN""" cherrypy.response.headers['Cache-control'] = 'no-cache' cherrypy.response.headers['Expire'] = 0 user, _ = credentials() page = "" lfn = lfn.strip() spanid = spanId(lfn) page += """<span id="%s" name="%s">""" % (spanid, spanid) statCode = 0 stop_codes = [StatusCode.TRANSFER_FAILED, StatusCode.CANCELLED, StatusCode.REMOVED] try: statCode = self.setStat(user, lfn) if statCode == StatusCode.FAILED: # this happen when proxy is expired, need to look at a log page += "Request fails. " elif statCode == StatusCode.UNKNOWN: page += 'lfn status unknown. ' elif statCode == StatusCode.CANCELLED: page += 'Transfer is cancelled. ' elif statCode == StatusCode.REMOVED: page += 'lfn is removed. ' elif statCode and statCode not in stop_codes: page += self.templatepage('templateLoading', msg="") page += self.updatePageWithLfnInfo(user, lfn) except Exception as exc: page += handleExc(exc) page += "</span>" return page @expose def download(self, *args, **_kwargs): """Server FileMover download area""" ctype = 'application/octet-stream' path = '%s/%s/%s' % (self.download_dir, args[0], args[1]) if os.path.isfile(path): return serve_file(path, content_type=ctype) raise HTTPError(500, 'File not found') @expose def images(self, *args, **_kwargs): """ Serve static images. """ args = list(args) _scripts = check_scripts(args, self.imgmap, self.imgdir) mime_types = ['*/*', 'image/gif', 'image/png', 'image/jpg', 'image/jpeg'] accepts = cherrypy.request.headers.elements('Accept') for accept in accepts: if accept.value in mime_types and len(args) == 1 \ and self.imgmap.has_key(args[0]): image = self.imgmap[args[0]] # use image extension to pass correct content type ctype = 'image/%s' % image.split('.')[-1] cherrypy.response.headers['Content-type'] = ctype if os.path.isfile(image): return serve_file(image, content_type=ctype) raise HTTPError(500, 'Image file not found') @expose @tools.gzip() def yui(self, *args, **_kwargs): """ Serve YUI library. YUI files has disperse directory structure, so input args can be in a form of (build, container, container.js) which corresponds to a single YUI JS file build/container/container.js """ cherrypy.response.headers['Content-Type'] = \ ["text/css", "application/javascript"] args = ['/'.join(args)] # preserve YUI dir structure scripts = check_scripts(args, self.yuimap, self.yuidir) return self.serve_files(args, scripts, self.yuimap) @expose @tools.gzip() def css(self, *args, **_kwargs): """ Cat together the specified css files and return a single css include. Multiple files can be supplied in a form of file1&file2&file3 """ cherrypy.response.headers['Content-Type'] = "text/css" args = parse_args(args) scripts = check_scripts(args, self.cssmap, self.cssdir) return self.serve_files(args, scripts, self.cssmap, 'css', True) @expose @tools.gzip() def js(self, *args, **_kwargs): """ Cat together the specified js files and return a single js include. Multiple files can be supplied in a form of file1&file2&file3 """ cherrypy.response.headers['Content-Type'] = "application/javascript" args = parse_args(args) scripts = check_scripts(args, self.jsmap, self.jsdir) return self.serve_files(args, scripts, self.jsmap) def serve_files(self, args, scripts, _map, datatype='', minimize=False): """ Return asked set of files for JS, YUI, CSS. """ idx = "-".join(scripts) if idx not in self.cache.keys(): data = '' if datatype == 'css': data = '@CHARSET "UTF-8";' for script in args: path = os.path.join(sys.path[0], _map[script]) path = os.path.normpath(path) ifile = open(path) data = "\n".join ([data, ifile.read().\ replace('@CHARSET "UTF-8";', '')]) ifile.close() if datatype == 'css': set_headers("text/css") if minimize: self.cache[idx] = minify(data) else: self.cache[idx] = data return self.cache[idx]