def write(self, cacheId, content, memory=False, writeToFile=True, keepLock=False): filetool.directory(self._path) cacheFile = os.path.join(self._path, self.filename(cacheId)) if writeToFile: try: if not cacheFile in self._locked_files: self._locked_files.add( cacheFile) # this is not atomic with the next one! filetool.lock(cacheFile) fobj = open(cacheFile, 'wb') pickle.dump(content, fobj, 2) fobj.close() if not keepLock: filetool.unlock(cacheFile) self._locked_files.remove( cacheFile) # not atomic with the previous one! except (IOError, EOFError, pickle.PickleError, pickle.PicklingError), e: e.args = ("Could not store cache to %s\n" % self._path + e.args[0], ) + e.args[1:] raise e
def write(self, cacheId, content, memory=False, writeToFile=True): filetool.directory(self._path) cacheFile = os.path.join(self._path, self.filename(cacheId)) if writeToFile: try: self._locked_files.add(cacheFile) # this is not atomic with the next one! filetool.lock(cacheFile) fobj = open(cacheFile, 'wb') pickle.dump(content, fobj, 2) fobj.close() filetool.unlock(cacheFile) self._locked_files.remove(cacheFile) # not atomic with the previous one! #if cacheId.startswith("tree-"): # print "caching: %s" % cacheId except (IOError, EOFError, pickle.PickleError, pickle.PicklingError): self._console.error("Could not store cache to %s" % self._path) sys.exit(1) if memory: memcache[cacheId] = content
def _handleResources(script, generator, filtered=True): def createResourceInfo(res, resval): resinfo = [ { "target": "resource", "data": { res : resval }} ] #filetool.save(approot+"/data/resource/" + res + ".json", json.dumpsCode(resinfo)) return resinfo def copyResource(res, library): sourcepath = os.path.join(library._resourcePath, res) targetpath = approot + "/resource/" + res filetool.directory(os.path.dirname(targetpath)) shutil.copy(sourcepath, targetpath) return # ---------------------------------------------------------------------- context.console.info("Processing resources: ", False) approot = context.jobconf.get("provider/app-root", "./provider") filetool.directory(approot+"/data") filetool.directory(approot+"/resource") # quick copy of runLogResources, for fast results packages = script.packagesSorted() parts = script.parts variants = script.variants allresources = {} if filtered: # -- the next call is fake, just to populate package.data.resources! _ = generator._codeGenerator.generateResourceInfoCode(script, generator._settings, context.jobconf.get("library",[])) for packageId, package in enumerate(packages): allresources.update(package.data.resources) else: # get the main library mainlib = [x for x in script.libraries if x.namespace == script.namespace][0] reslist = mainlib.getResources() allresources = ResourceHandler.createResourceStruct(reslist, updateOnlyExistingSprites = False) # get resource info resinfos = {} numResources = len(allresources) for num,res in enumerate(allresources): context.console.progress(num+1, numResources) # fake a classId-like resourceId ("a.b.c"), for filter matching resId = os.path.splitext(res)[0] resId = resId.replace("/", ".") if filtered and not passesOutputfilter(resId): continue resinfos[res] = createResourceInfo(res, allresources[res]) # extract library name space if isinstance(allresources[res], types.ListType): # it's an image = [14, 14, u'png', u'qx' [, u'qx/decoration/Modern/checkradio-combined.png', 0, 0]] library_ns = allresources[res][3] else: # html page etc. = "qx" library_ns = allresources[res] if library_ns: # library_ns == '' means embedded image -> no copying library = libraries[library_ns] copyResource(res, library) filetool.save(approot+"/data/resource/resources.json", json.dumpsCode(resinfos)) return
def write(self, cacheId, content, memory=False, writeToFile=True, keepLock=False): filetool.directory(self._path) cacheFile = os.path.join(self._path, self.filename(cacheId)) if writeCond(cacheId): print "\nWriting %s ..." % (cacheId,), if writeToFile: try: if not cacheFile in self._locked_files: self._locked_files.add(cacheFile) # this is not atomic with the next one! filetool.lock(cacheFile) fobj = open(cacheFile, 'wb') pickle.dump(content, fobj, 2) fobj.close() if not keepLock: filetool.unlock(cacheFile) self._locked_files.remove(cacheFile) # not atomic with the previous one! #print "wrote cacheId: %s" % cacheId if writeCond(cacheId): print "to disk" except (IOError, EOFError, pickle.PickleError, pickle.PicklingError), e: e.args = ("Could not store cache to %s\n" % self._path + e.args[0], ) + e.args[1:] raise e
def read(self, cacheId, dependsOn=None, memory=False): if memcache.has_key(cacheId): return memcache[cacheId] filetool.directory(self._path) cacheFile = os.path.join(self._path, self.filename(cacheId)) try: cacheModTime = os.stat(cacheFile).st_mtime except OSError: return None # Out of date check if dependsOn: fileModTime = os.stat(dependsOn).st_mtime if fileModTime > cacheModTime: return None try: content = cPickle.load(open(cacheFile, 'rb')) if memory: memcache[cacheId] = content return content except (IOError, EOFError, cPickle.PickleError, cPickle.UnpicklingError): self._console.error("Could not read cache from %s" % self._path) return None
def archive_download(self, url, cache_path, checksum): rc = 0 # Download arcfile = os.path.join(cache_path, os.path.basename(url)) tdir = os.path.dirname(arcfile) filetool.directory(tdir) tfp = open(arcfile, "wb") #(fname, urlinfo) = urllib.urlretrieve(url, arcfile) urlobj = urllib.urlopen(url) assert urlobj.getcode() == 200, "Could not the download contrib archive: %s" % url hashobj = self.copy_and_hash(urlobj.fp, tfp) assert hashobj.hexdigest()==checksum, "Checksum of archive does not validate (should be: %s): %s" % (checksum, arcfile) urlobj.close() tfp.close() # Extract if url.endswith('.zip'): zipf = ZipFile(arcfile, 'r') zipf.extractall(tdir) zipf.close() else: # .tar, .tgz, .tar.gz, .tar.bz2 tar = tarfile.open(arcfile) tar.extractall(tdir) tar.close # Eliminate archive top-dir _, archive_dirs, _ = os.walk(tdir).next() assert archive_dirs, "The downloaded archive is not in single top-dir format: %s" % arcfile archive_top = os.path.join(tdir, archive_dirs[0]) # just take the first dir entry for item in os.listdir(archive_top): shutil.move(os.path.join(archive_top, item), tdir) os.rmdir(archive_top) os.unlink(arcfile) return rc
def read(self, cacheId, dependsOn=None, memory=False, keepLock=False): if dependsOn: dependsModTime = os.stat(dependsOn).st_mtime if writeCond(cacheId): print "\nReading %s ..." % (cacheId, ), # Mem cache if cacheId in memcache: memitem = memcache[cacheId] if not dependsOn or dependsModTime < memitem['time']: if writeCond(cacheId): print "from memcache" return memitem['content'], memitem['time'] # File cache filetool.directory(self._path) cacheFile = os.path.join(self._path, self.filename(cacheId)) try: cacheModTime = os.stat(cacheFile).st_mtime except OSError: return None, None # out of date check if dependsOn and dependsModTime > cacheModTime: return None, cacheModTime try: if not cacheFile in self._locked_files: self._locked_files.add(cacheFile) filetool.lock(cacheFile) fobj = open(cacheFile, 'rb') #filetool.lock(fobj.fileno()) gc.disable() try: content = pickle.loads(fobj.read().decode('zlib')) finally: gc.enable() #filetool.unlock(fobj.fileno()) fobj.close() if not keepLock: filetool.unlock(cacheFile) self._locked_files.remove(cacheFile) if memory: memcache[cacheId] = {'content': content, 'time': time.time()} #print "read cacheId: %s" % cacheId if writeCond(cacheId): print "from disk" return content, cacheModTime except (IOError, EOFError, pickle.PickleError, pickle.UnpicklingError): self._console.warn( "Could not read cache object %s, recalculating..." % cacheFile) return None, cacheModTime
def _check_path(self, path): if not os.path.exists(path): filetool.directory(path) elif not os.path.isdir(path): raise RuntimeError, "The cache path is not a directory: %s" % path else: # it's an existing directory # defer read/write access to the first call of read()/write() pass
def combine(self, combined, files, horizontal): self._console.indent() montage_cmd = "montage -geometry +0+0 -gravity NorthWest -tile %s -background None %s %s" if horizontal: orientation = "x1" else: orientation = "1x" # combine config = [] clips = [] top = 0 left = 0 allfiles = [] for file in files: allfiles.extend(glob.glob(file)) #self._console.debug("Combining the following images: %r" % allfiles) for file in allfiles: if not os.path.exists(file): self._console.warn("Non-existing file spec, skipping: %s" % file) continue clips.append(file) imginfo = self._imageInfo.getImageInfo(file, file) width, height = imginfo['width'], imginfo['height'] config.append({ 'file': file, 'combined': combined, 'left': -left, 'top': -top, 'width': width, 'height': height, 'type': imginfo['type'] }) if horizontal: left += width else: top += height if len(clips) == 0: self._console.warn("No images to combine; skipping") else: filetool.directory(os.path.dirname(combined)) (fileDescriptor, tempPath) = tempfile.mkstemp(text=True, dir=os.curdir) temp = os.fdopen(fileDescriptor, "w") temp.write("\n".join(clips)) temp.close() cmd = montage_cmd % (orientation, "@" + os.path.basename(tempPath), combined) rc = os.system(cmd) os.unlink(tempPath) if rc != 0: raise RuntimeError, "The montage command (%s) failed with the following return code: %d" % ( cmd, rc) self._console.outdent() return config
def read(self, cacheId, dependsOn=None, memory=False, keepLock=False): if dependsOn: dependsModTime = os.stat(dependsOn).st_mtime if writeCond(cacheId): print "\nReading %s ..." % (cacheId,), # Mem cache if cacheId in memcache: memitem = memcache[cacheId] if not dependsOn or dependsModTime < memitem['time']: if writeCond(cacheId): print "from memcache" return memitem['content'], memitem['time'] # File cache filetool.directory(self._path) cacheFile = os.path.join(self._path, self.filename(cacheId)) try: cacheModTime = os.stat(cacheFile).st_mtime except OSError: return None, None # out of date check if dependsOn and dependsModTime > cacheModTime: return None, cacheModTime try: if not cacheFile in self._locked_files: self._locked_files.add(cacheFile) filetool.lock(cacheFile) fobj = open(cacheFile, 'rb') #filetool.lock(fobj.fileno()) gc.disable() try: content = pickle.loads(fobj.read().decode('zlib')) finally: gc.enable() #filetool.unlock(fobj.fileno()) fobj.close() if not keepLock: filetool.unlock(cacheFile) self._locked_files.remove(cacheFile) if memory: memcache[cacheId] = {'content':content, 'time': time.time()} #print "read cacheId: %s" % cacheId if writeCond(cacheId): print "from disk" return content, cacheModTime except (IOError, EOFError, pickle.PickleError, pickle.UnpicklingError): self._console.warn("Could not read cache object %s, recalculating..." % cacheFile) return None, cacheModTime
def combine(self, combined, files, horizontal): self._console.indent() montage_cmd = "montage -geometry +0+0 -gravity NorthWest -tile %s -background None %s %s" if horizontal: orientation = "x1" else: orientation = "1x" # combine config = [] clips = [] top = 0 left = 0 allfiles = [] for file in files: allfiles.extend(glob.glob(file)) # self._console.debug("Combining the following images: %r" % allfiles) for file in allfiles: if not os.path.exists(file): self._console.warn("Non-existing file spec, skipping: %s" % file) continue clips.append(file) imginfo = self._imageInfo.getImageInfo(file, file) width, height = imginfo["width"], imginfo["height"] config.append( { "file": file, "combined": combined, "left": -left, "top": -top, "width": width, "height": height, "type": imginfo["type"], } ) if horizontal: left += width else: top += height if len(clips) == 0: self._console.warn("No images to combine; skipping") else: filetool.directory(os.path.dirname(combined)) (fileDescriptor, tempPath) = tempfile.mkstemp(text=True, dir=os.curdir) temp = os.fdopen(fileDescriptor, "w") temp.write("\n".join(clips)) temp.close() cmd = montage_cmd % (orientation, "@" + os.path.basename(tempPath), combined) rc = os.system(cmd) os.unlink(tempPath) if rc != 0: raise RuntimeError, "The montage command (%s) failed with the following return code: %d" % (cmd, rc) self._console.outdent() return config
def copyResource(res, library): if skip_expression.search(os.path.basename(res)): return sourcepath = os.path.join(library.resourcePath, res) targetpath = approot + "/resource/" + res filetool.directory(os.path.dirname(targetpath)) shutil.copy(sourcepath, targetpath) #copier = copytool.CopyTool(context.console) #args = ['-x', ','.join(filetool.VERSIONCONTROL_DIR_PATTS), sourcepath, targetpath] #copier.parse_args(args) #copier.do_work() return
def main(): apidata = {} apidata['type'] = 'doctree' apidata['children'] = [] apidata['attributes'] = {} apidata['children'].append({ "type": "packages", "attributes": {}, "children": [] }) filetool.directory(store_path) dirwalker = filetool.find(module_root, r'\.py$') for pyfile in dirwalker: #if os.stat(pyfile).st_size == 0: # continue # get the file's api doc as json filejson = pyapi2json(pyfile) apipackage = file2package(pyfile, module_root) # and store it filetool.save(store_path + '/' + apipackage + '.json', filejson) # make an entry in apidata struct levels = apipackage.split('.') curr = apidata['children'][0]['children'] for pos, level in enumerate(levels): if level not in (x['attributes']['name'] for x in curr if 'name' in x['attributes']): newentry = { "children": [], "type": "packages" if pos % 2 else "package", "attributes": { "packageName": ".".join(levels[:pos]), "name": level, "fullName": ".".join(levels[:pos + 1]) } } if pos == len(levels) - 1: newentry["externalRef"] = True #del newentry['children'] #newentry["type"] = "classes" pass curr.append(newentry) curr = newentry['children'] else: curr = [ x['children'] for x in curr if x['attributes']['name'] == level ][0] # store apidata filetool.save(store_path + '/' + "apidata.json", json.dumps(apidata))
def _check_path(self, path): self._console.indent() self._console.debug("Checking path '%s'" % path) if not os.path.exists(path): self._console.debug("Creating non-existing cache directory") filetool.directory(path) self._update_checkfile() elif not os.path.isdir(path): raise RuntimeError, "The cache path is not a directory: %s" % path else: # it's an existing directory # defer read/write access test to the first call of read()/write() self._console.debug("Using existing directory") pass self._console.outdent()
def write(self, cacheId, content, memory=False, writeToFile=True): filetool.directory(self._path) cacheFile = os.path.join(self._path, self.filename(cacheId)) if writeToFile: try: cPickle.dump(content, open(cacheFile, 'wb'), 2) except (IOError, EOFError, cPickle.PickleError, cPickle.PicklingError): self._console.error("Could not store cache to %s" % self._path) sys.exit(1) if memory: memcache[cacheId] = content
def combine(self, combined, files, horizontal, type="extension"): self._console.indent() if horizontal: orientation = "x1" else: orientation = "1x" # combine config = [] clips = [] top = 0 left = 0 allfiles = [] for file in files: allfiles.extend(glob.glob(file)) #self._console.debug("Combining the following images: %r" % allfiles) for file in allfiles: if not os.path.exists(file): self._console.warn("Non-existing file spec, skipping: %s" % file) continue clips.append(file) imginfo = Image(file).getInfoMap() width, height = imginfo['width'], imginfo['height'] config.append({ 'file': file, 'combined': combined, 'left': -left, 'top': -top, 'width': width, 'height': height, 'type': imginfo['type'] }) if horizontal: left += width else: top += height if len(clips) == 0: self._console.warn("No images to combine; skipping") else: filetool.directory(os.path.dirname(combined)) if type == "extension": self.combineImgMagick(clips, combined, orientation) elif type == "base64": self.combineBase64(config) self._console.outdent() return config
def _handleResources(script, generator): def createResourceInfo(res, resval): resinfo = [ { "target": "resource", "data": { res : resval }} ] #filetool.save(approot+"/data/resource/" + res + ".json", json.dumpsCode(resinfo)) return resinfo def copyResource(res, library): sourcepath = os.path.join(library['path'], library['resource'], res) targetpath = approot + "/resource/" + res filetool.directory(os.path.dirname(targetpath)) shutil.copy(sourcepath, targetpath) return # ---------------------------------------------------------------------- approot = context.jobconf.get("provider/app-root", "./provider") filetool.directory(approot+"/data") filetool.directory(approot+"/resource") # quick copy of runLogResources, for fast results packages = script.packagesSortedSimple() parts = script.parts variants = script.variants allresources = {} # get resource info # -- the next call is fake, just to populate package.data.resources! _ = generator._codeGenerator.generateResourceInfoCode(script, generator._settings, context.jobconf.get("library",[])) for packageId, package in enumerate(packages): allresources.update(package.data.resources) resinfos = {} for res in allresources: # fake a classId-like resourceId ("a.b.c"), for filter matching resId = os.path.splitext(res)[0] resId = resId.replace("/", ".") if passesOutputfilter(resId): resinfos[res] = createResourceInfo(res, allresources[res]) # extract library name space if isinstance(allresources[res], types.ListType): # it's an image = [14, 14, u'png', u'qx' [, u'qx/decoration/Modern/checkradio-combined.png', 0, 0]] library_ns = allresources[res][3] else: # html page etc. = "qx" library_ns = allresources[res] library = libraries[library_ns] copyResource(res, library) filetool.save(approot+"/data/resource/resources.json", json.dumpsCode(resinfos)) return
def _check_path(self, path): self._console.indent() self._console.debug("Checking path '%s'" % path) if not os.path.exists(path): self._console.debug("Creating non-existing cache directory") filetool.directory(path) self._update_checkfile() elif not os.path.isdir(path): raise RuntimeError, "The cache path is not a directory: %s" % path else: # it's an existing directory # defer read/write access test to the first call of read()/write() self._console.debug("Using existing directory") pass if len(os.listdir(path)) < CACHE_THRESHOLD: # not even minimal framework classes cached self._console.info("Populating the cache, this may take some time") self._console.outdent()
def read(self, cacheId, dependsOn=None, memory=False, keepLock=False): if dependsOn: dependsModTime = os.stat(dependsOn).st_mtime # Mem cache if cacheId in memcache: memitem = memcache[cacheId] if not dependsOn or dependsModTime < memitem['time']: return memitem['content'], memitem['time'] # File cache filetool.directory(self._path) cacheFile = os.path.join(self._path, self.filename(cacheId)) try: cacheModTime = os.stat(cacheFile).st_mtime except OSError: return None, None # out of date check if dependsOn and dependsModTime > cacheModTime: return None, cacheModTime try: if not cacheFile in self._locked_files: self._locked_files.add(cacheFile) filetool.lock(cacheFile) fobj = open(cacheFile, 'rb') #filetool.lock(fobj.fileno()) content = pickle.load(fobj) #filetool.unlock(fobj.fileno()) fobj.close() if not keepLock: filetool.unlock(cacheFile) self._locked_files.remove(cacheFile) if memory: memcache[cacheId] = {'content': content, 'time': time.time()} return content, cacheModTime except (IOError, EOFError, pickle.PickleError, pickle.UnpicklingError): self._console.error("Could not read cache from %s" % self._path) return None, cacheModTime
def main(): apidata = {} apidata['type'] = 'doctree' apidata['children'] = [] apidata['attributes'] = {} apidata['children'].append({ "type":"packages","attributes":{},"children":[] }) filetool.directory(store_path) dirwalker = filetool.find(module_root, r'\.py$') for pyfile in dirwalker: #if os.stat(pyfile).st_size == 0: # continue # get the file's api doc as json filejson = pyapi2json(pyfile) apipackage = file2package(pyfile, module_root) # and store it filetool.save(store_path+'/'+apipackage+'.json', filejson) # make an entry in apidata struct levels = apipackage.split('.') curr = apidata['children'][0]['children'] for pos,level in enumerate(levels): if level not in (x['attributes']['name'] for x in curr if 'name' in x['attributes']): newentry = { "children" : [], "type" : "packages" if pos % 2 else "package", "attributes" : { "packageName" : ".".join(levels[:pos]), "name" : level, "fullName" : ".".join(levels[:pos+1]) } } if pos==len(levels)-1: newentry["externalRef"] = True #del newentry['children'] #newentry["type"] = "classes" pass curr.append(newentry) curr = newentry['children'] else: curr = [x['children'] for x in curr if x['attributes']['name']==level][0] # store apidata filetool.save(store_path+'/'+"apidata.json", json.dumps(apidata))
def read(self, cacheId, dependsOn=None, memory=False, keepLock=False): if dependsOn: dependsModTime = os.stat(dependsOn).st_mtime # Mem cache if cacheId in memcache: memitem = memcache[cacheId] if not dependsOn or dependsModTime < memitem['time']: return memitem['content'], memitem['time'] # File cache filetool.directory(self._path) cacheFile = os.path.join(self._path, self.filename(cacheId)) try: cacheModTime = os.stat(cacheFile).st_mtime except OSError: return None, None # out of date check if dependsOn and dependsModTime > cacheModTime: return None, cacheModTime try: if not cacheFile in self._locked_files: self._locked_files.add(cacheFile) filetool.lock(cacheFile) fobj = open(cacheFile, 'rb') #filetool.lock(fobj.fileno()) content = pickle.load(fobj) #filetool.unlock(fobj.fileno()) fobj.close() if not keepLock: filetool.unlock(cacheFile) self._locked_files.remove(cacheFile) if memory: memcache[cacheId] = {'content':content, 'time': time.time()} return content, cacheModTime except (IOError, EOFError, pickle.PickleError, pickle.UnpicklingError): self._console.error("Could not read cache from %s" % self._path) return None, cacheModTime
def _handleCode(script, generator): approot = context.jobconf.get("provider/app-root", "./provider") filetool.directory(approot + "/code") for clazz in script.classesObj: # register library (for _handleResources) if clazz.library['namespace'] not in libraries: libraries[clazz.library['namespace']] = clazz.library if passesOutputfilter(clazz.id, ): classAId = clazz.id.replace(".","/") + ".js" sourcepath = os.path.join(clazz.library['path'], clazz.library['class'], classAId) # TODO: this should be a class method targetpath = approot + "/code/" + classAId filetool.directory(os.path.dirname(targetpath)) shutil.copy(sourcepath, targetpath) return
def write(self, cacheId, content, memory=False, writeToFile=True, keepLock=False): filetool.directory(self._path) cacheFile = os.path.join(self._path, self.filename(cacheId)) if writeCond(cacheId): print "\nWriting %s ..." % (cacheId, ), if writeToFile: try: if not cacheFile in self._locked_files: self._locked_files.add( cacheFile) # this is not atomic with the next one! filetool.lock(cacheFile) fobj = open(cacheFile, 'wb') pickle.dump(content, fobj, 2) fobj.close() if not keepLock: filetool.unlock(cacheFile) self._locked_files.remove( cacheFile) # not atomic with the previous one! #print "wrote cacheId: %s" % cacheId if writeCond(cacheId): print "to disk" except (IOError, EOFError, pickle.PickleError, pickle.PicklingError), e: try: os.unlink( cacheFile ) # try remove cache file, Pickle might leave incomplete files except: e.args = ("Cache file might be crippled.\n" % self._path + e.args[0], ) + e.args[1:] e.args = ("Could not store cache to %s.\n" % self._path + e.args[0], ) + e.args[1:] raise e
def write(self, cacheId, content, memory=False, writeToFile=True): filetool.directory(self._path) cacheFile = os.path.join(self._path, self.filename(cacheId)) if writeToFile: try: self._locked_files.add(cacheFile) # this is not atomic with the next one! filetool.lock(cacheFile) fobj = open(cacheFile, 'wb') pickle.dump(content, fobj, 2) fobj.close() filetool.unlock(cacheFile) self._locked_files.remove(cacheFile) # not atomic with the previous one! except (IOError, EOFError, pickle.PickleError, pickle.PicklingError), e: e.args = ("Could not store cache to %s\n" % self._path + e.args[0], ) + e.args[1:] raise e
def combine(self, combined, files, horizontal, type="extension"): self._console.indent() if horizontal: orientation = "x1" else: orientation = "1x" # combine config = [] clips = [] top = 0 left = 0 allfiles = [] for file in files: allfiles.extend(glob.glob(file)) #self._console.debug("Combining the following images: %r" % allfiles) for file in allfiles: if not os.path.exists(file): self._console.warn("Non-existing file spec, skipping: %s" % file) continue clips.append(file) imginfo = Image(file).getInfoMap() width, height = imginfo['width'], imginfo['height'] config.append({'file':file, 'combined':combined, 'left': -left, 'top': -top, 'width':width, 'height':height, 'type':imginfo['type']}) if horizontal: left += width else: top += height if len(clips) == 0: self._console.warn("No images to combine; skipping") else: filetool.directory(os.path.dirname(combined)) if type == "extension": self.combineImgMagick(clips, combined, orientation) elif type == "base64": self.combineBase64(config) self._console.outdent() return config
def read(self, cacheId, dependsOn=None, memory=False): if memcache.has_key(cacheId): return memcache[cacheId] filetool.directory(self._path) cacheFile = os.path.join(self._path, self.filename(cacheId)) try: cacheModTime = os.stat(cacheFile).st_mtime except OSError: return None # Out of date check if dependsOn: fileModTime = os.stat(dependsOn).st_mtime if fileModTime > cacheModTime: return None try: self._locked_files.add(cacheFile) filetool.lock(cacheFile) fobj = open(cacheFile, 'rb') #filetool.lock(fobj.fileno()) content = pickle.load(fobj) #filetool.unlock(fobj.fileno()) fobj.close() filetool.unlock(cacheFile) self._locked_files.remove(cacheFile) if memory: memcache[cacheId] = content return content except (IOError, EOFError, pickle.PickleError, pickle.UnpicklingError): self._console.error("Could not read cache from %s" % self._path) return None
def archive_download(self, url, cache_path, checksum): rc = 0 # Download arcfile = os.path.join(cache_path, os.path.basename(url)) tdir = os.path.dirname(arcfile) filetool.directory(tdir) tfp = open(arcfile, "wb") #(fname, urlinfo) = urllib.urlretrieve(url, arcfile) urlobj = urllib.urlopen(url) assert urlobj.getcode( ) == 200, "Could not the download contrib archive: %s" % url hashobj = self.copy_and_hash(urlobj.fp, tfp) assert hashobj.hexdigest( ) == checksum, "Checksum of archive does not validate (should be: %s): %s" % ( checksum, arcfile) urlobj.close() tfp.close() # Extract if url.endswith('.zip'): zipf = ZipFile(arcfile, 'r') zipf.extractall(tdir) zipf.close() else: # .tar, .tgz, .tar.gz, .tar.bz2 tar = tarfile.open(arcfile) tar.extractall(tdir) tar.close # Eliminate archive top-dir _, archive_dirs, _ = os.walk(tdir).next() assert archive_dirs, "The downloaded archive is not in single top-dir format: %s" % arcfile archive_top = os.path.join( tdir, archive_dirs[0]) # just take the first dir entry for item in os.listdir(archive_top): shutil.move(os.path.join(archive_top, item), tdir) os.rmdir(archive_top) os.unlink(arcfile) return rc
def indexAll(options): if options.cacheDirectory != None: filetool.directory(options.cacheDirectory) print " * Indexing class paths... " fileDb = {} moduleDb = {} listIndex = 0 for classPath in options.classPath: print " - Indexing: %s" % classPath counter = indexClassPath(classPath, listIndex, options, fileDb, moduleDb) print " - %s classes were found" % counter listIndex += 1 # Resolving auto-deps resolveAutoDeps(fileDb, options) if options.cacheDirectory != None: storeEntryCache(fileDb, options) return fileDb, moduleDb
def _handleCode(script, generator): approot = context.jobconf.get("provider/app-root", "./provider") builds = context.jobconf.get("provider/compile", ["source"]) for buildtype in builds: context.console.info("Processing %s version of classes:\t" % buildtype, False) if buildtype == "source": targetdir = approot + "/code" filetool.directory(targetdir) elif buildtype == "build": targetdir = approot + "/code-build" filetool.directory(targetdir) optimize = context.jobconf.get( "compile-options/code/optimize", ["variables", "basecalls", "strings"]) variantsettings = context.jobconf.get("variants", {}) variantSets = util.computeCombinations(variantsettings) else: raise ConfigurationError("Unknown provider compile type '%s'" % buildtype) numClasses = len(script.classesObj) for num, clazz in enumerate(script.classesObj): context.console.progress(num + 1, numClasses) # register library (for _handleResources) if clazz.library.namespace not in libraries: libraries[clazz.library.namespace] = clazz.library if passesOutputfilter(clazz.id, ): classAId = clazz.id.replace(".", "/") + ".js" targetpath = targetdir + "/" + classAId filetool.directory(os.path.dirname(targetpath)) if buildtype == "source": shutil.copy(clazz.path, targetpath) elif buildtype == "build": compOptions = CompileOptions( optimize, variantSets[0] ) # only support for a single variant set! code = clazz.getCode(compOptions) filetool.save(targetpath, code) return
def _handleCode(script, generator): approot = context.jobconf.get("provider/app-root", "./provider") builds = context.jobconf.get("provider/compile", ["source"]) for buildtype in builds: context.console.info("Processing %s version of classes:\t" % buildtype, False) if buildtype == "source": targetdir = approot + "/code" filetool.directory(targetdir) elif buildtype == "build": targetdir = approot + "/code-build" filetool.directory(targetdir) optimize = context.jobconf.get("compile-options/code/optimize", ["variables","basecalls","strings"]) variantsettings = context.jobconf.get("variants", {}) variantSets = util.computeCombinations(variantsettings) else: raise ConfigurationError("Unknown provider compile type '%s'" % buildtype) numClasses = len(script.classesObj) for num, clazz in enumerate(script.classesObj): context.console.progress(num+1, numClasses) # register library (for _handleResources) if clazz.library.namespace not in libraries: libraries[clazz.library.namespace] = clazz.library if passesOutputfilter(clazz.id, ): classAId = clazz.id.replace(".","/") + ".js" targetpath = targetdir + "/" + classAId filetool.directory(os.path.dirname(targetpath)) if buildtype == "source": shutil.copy(clazz.path, targetpath) elif buildtype == "build": compOptions = CompileOptions(optimize, variantSets[0]) # only support for a single variant set! code = clazz.getCode(compOptions) filetool.save(targetpath, code) return
def saveFile(self, path, cont): filetool.directory(os.path.dirname(path)) if not path.endswith(self.urlPathSep): fo = open(path, 'wb') fo.write(cont) fo.close
def saveFile(self, path, cont): filetool.directory(os.path.dirname(path)) fo = open(path, 'wb') fo.write(cont) fo.close
def copyResource(res, library): sourcepath = os.path.join(library._resourcePath, res) targetpath = approot + "/resource/" + res filetool.directory(os.path.dirname(targetpath)) shutil.copy(sourcepath, targetpath) return
# # http://qooxdoo.org # # Copyright: # 2006-2013 1&1 Internet AG, Germany, http://www.1und1.de # # License: # MIT: https://opensource.org/licenses/MIT # See the LICENSE file in the project's top-level directory for details. # # Authors: # * Thomas Herchenroeder (thron7) # ################################################################################ ## # NAME # mkdir.py -- a cross-platform mkdir # # SYNTAX # python mkdir.py <path> -- create path directory, pot. with intervening dirs # ## import os, sys import qxenviron from misc import filetool filetool.directory(sys.argv[1])