def get_all_entries_as_paths(self):
r = []
for top in listdir(structure.UPLOADS_DIR):
for middle in listdir(join(structure.UPLOADS_DIR,top)):
for ipnumber in listdir(join(structure.UPLOADS_DIR,top,middle)):
r.append(IdentityPath(top,middle,ipnumber))
return r
def _generate_assets(self,**options):
filenames = []
media_target_dir = os.path.join(structure.TARGET_DIR,"mediasite")
makedirs(os.path.join(media_target_dir,"css"))
makedirs(os.path.join(media_target_dir,"js"))
for d in listdir(structure.CSS_DIR):
if os.path.splitext(d)[1] == ".css" and os.path.isdir(os.path.join(structure.CSS_DIR,d)):
filenames.append(d)
target = os.path.join(structure.MEDIASITE_DIRS[-1],"css",d)
src = os.path.join(structure.CSS_DIR,d)
sources = listdir(src,full_path=True,recursed=True)
lines = combine_asset_sources(sources,structure.CSS_DIR,source_node=CssSourceNode)
with open(os.path.join(media_target_dir, "css", d), 'w') as handle:
handle.write(''.join(lines))
handle.flush()
for d in listdir(structure.JS_DIR):
if os.path.splitext(d)[1] == ".js" and os.path.isdir(os.path.join(structure.JS_DIR,d)):
filenames.append(d)
target = os.path.join(structure.MEDIASITE_DIRS[-1],"js",d)
src = os.path.join(structure.JS_DIR,d)
sources = listdir(src,full_path=True,recursed=True)
lines = combine_asset_sources(sources,structure.JS_DIR,source_node=JsSourceNode)
with open(os.path.join(media_target_dir, "js", d), 'w') as handle:
handle.write(''.join(lines))
handle.flush()
print "Generating", ' '.join(filenames), '->', media_target_dir
def entities(cls,path):
r = []
for middle in listdir(structure.MEDIASITE_DIRS[0],filters=(filters.only_directories,)): #TODO exclude js,css,embed
if middle in ('images',):
for ipnumber in listdir(join(structure.MEDIASITE_DIRS[0],middle),filters=(filters.only_directories,)):
entity = StaticEntity(IdentityPath("static",middle,ipnumber))
#TODO verify entity
r.append(entity)
return r
def entities(cls,path):
r = []
for top in listdir(structure.UPLOADS_DIR,filters=(filters.only_directories,)): #TODO restrict mediasite and only known media files
for middle in listdir(join(structure.UPLOADS_DIR,top),filters=(filters.only_directories,)):
for ipnumber in listdir(join(structure.UPLOADS_DIR,top,middle),filters=(filters.only_directories,)):
entity = StaticEntity(IdentityPath(top,middle,ipnumber))
#TODO verify entity
r.append(entity)
return r
def _get_original(self):
if not hasattr(self,"_original") or not self._original:
uploads = [f for f in listdir(self.upload_directory) if not f.startswith(".") and not f in IGNORE_IN_UPLOADS]
if len(uploads) == 0:
return None
self._original = AssetFile(self,join(self.upload_directory,uploads[0]))
return self._original
def __init__(self,path=None,parent=None,file_name=None):
"""Create asset based on the original file path
file_name can be with or without extension
top,area,file_name = path"""
if path:
st,top,area,file_name = StaticPath(path)
else:
st,top,area = StaticPath(parent.path)
self.upload_directory = join(structure.MEDIASITE_DIRS[0],top,area)
if not exists(self.upload_directory):
self.original = None
return #TODO use unsupported file
name,ext = splitext(file_name)
if ext is '':
originals = fs.listdir(self.upload_directory,filters=(fs.filters.fnmatch(file_name+".*"),fs.filters.no_hidden,fs.filters.no_system))
if len(originals)==0:
self.original = None
return #TODO use unsupported file
file_name = originals[0]
self.download_directory = join(structure.DOWNLOADS_DIR,"static",top,area,name)
fs.makedirs(self.download_directory)
self.original = AssetFile(self,join(self.upload_directory,file_name))
self.original_category = self.original.filetype[1]
self.caption = self.original.caption
self.path = "/%s/%s/%s/%s" % ('static',top,area,name)
def load_scopes():
from thepian.conf import structure
base = join(structure.JS_DIR)
scopes = [s for s in listdir(base,filters=(filters.fnmatch("*.scope.js"),))]
for s in scopes:
load_and_add_scope('"%s"' % s[:-9],join(base,s))
print "Done loading scopes (", " ".join(scopes),")"
def list_entities_as_paths(self):
r = []
if self.paths[0].static_path:
base = join(structure.MEDIASITE_DIRS[0],self.paths[0].path[1])
else:
base = join(structure.UPLOADS_DIR, self.paths[0].path[0], self.paths[0].path[1])
for name in listdir(base,filters=(filters.only_directories,)):
r.append(IdentityPath(self.paths[0].path[0], self.paths[0].path[1], name ))
return r
def get(self,account,project,file_name_bit):
import zipfile
import StringIO
info = describe_project(account,project)
if not info:
raise tornado.web.HTTPError(404)
node_id = getNodeId(self,account, project) # node cookie
upload_id = info["uploads"][0]["hash"]
spec_upload_url = "%s://%s/%s/%s/%s" % (self.request.protocol,self.request.host,project,upload_id,self.upload_script_name)
js_base = join(structure.TEMPLATES_DIR,"pagespec","example","js")
js_list = listdir(js_base,filters=(filters.no_hidden,filters.no_system,filters.no_directories))
tdata = {
"project": project,
"account": account,
"spec_upload_url": spec_upload_url,
}
introduction = self.render_string("pagespec/example/introduction.html", **tdata)
index = self.render_string("pagespec/example/spec/index.html", **tdata)
example_spec = self.render_string("pagespec/example/spec/example.spec.js")
in_memory_zip = StringIO.StringIO()
zf = zipfile.ZipFile(in_memory_zip, 'w', zipfile.ZIP_DEFLATED, False)
zf.writestr('introduction.html', introduction)
zf.writestr('spec/index.html', index)
zf.writestr('spec/example.spec.js',example_spec)
for suite in info["suites"]:
tdata["suite_id"] = suite["hash"]
tdata["pipeline_id"] = suite["hash"]
tdata["suite"] = suite["suite"]
tdata["exec_name"] = suite["exec_name"]
suite_script_name = "%s-suite-%s" % (suite["suite"],suite["exec_name"])
tdata["suite_script_url"] = "%s://%s/%s/%s/%s.js" % (self.request.protocol,self.request.host,project,suite["hash"],suite_script_name)
runner = self.render_string("pagespec/example/suite-runner.html", **tdata)
zf.writestr("%s-suite-runner.html" % str(suite["suite"]), runner)
for js_file in js_list:
contents = self.render_string(join(js_base,js_file), **tdata)
zf.writestr(join("js",js_file),contents)
for zfile in zf.filelist:
zfile.create_system = 0
zf.close()
self.set_header("Content-Type","application/octet-stream")
self.write(in_memory_zip.getvalue())
self.flush()
def test_exclude():
apply_site_dirs("",force_project_path=join(pages_test_root,"web"))
options = ObjectLike({
"source": None,
"dest": "_site",
"fork": True,
"appcache": False,
"debug": False,
"pygments": False
})
config = SiteConfig(options)
from fs import filters, listdir
base_filters = [
config.exclude_filter(),
filters.no_hidden,filters.no_system]
r = listdir(site.SITE_DIR,recursed=True,filters=base_filters)
print >>sys.stderr, r
assert r == ["abc_shim.js","index.md","unicode.md","js/init.js"]
def refresh(self):
"""Create download/video.* audio.* imageN.*
returns: mimetype, category, supported, bases_directory, thumbnail_directory, path, image_bases, video_bases
"""
supported = False
category = "file"
mimetype = "unknown/unknown"
original = self.original
if original:
mimetype, category, supported = original.filetype
if not supported:
print "not supported ", category, mimetype, original.file_name
bases_directory = join(structure.MEDIASITE_DIRS[0],"icons","unsupported-%s" % category)
thumbnail_directory = join(structure.DOWNLOADS_DIR,"icons","unsupported-%s" % category)
path = "/%s/%s" % ('icons',"unsupported-%s" % category)
else:
bases_directory = thumbnail_directory = self.download_directory
path = self.path
if self.original.filetype == "image/png":
image0 = AssetFile(self,join(bases_directory,"image0.png"),thumbnail_directory=thumbnail_directory,path="%s/%s" % (path,"image0.png"))
else:
image0 = AssetFile(self,join(bases_directory,"image0.jpg"),thumbnail_directory=thumbnail_directory,path="%s/%s" % (path,"image0.jpg"))
if not image0.exists or newer(self.original.file_path,image0.file_path):
image0.make_base_image(self.original,0)
existing = listdir(bases_directory,filters=(filters.no_hidden,filters.no_system,filters.no_directories))
image_file_names = ("image0.jpg","image0.png","image1.jpg","image1.png","image2.jpg","image2.png","image3.jpg","image3.png","image4.jpg","image4.png","image5.jpg","image5.png","image6.jpg","image6.png","image7.jpg","image7.png","image8.jpg","image8.png","image9.jpg""image9.png")
video_file_names = ("video480.m4v","video480.flv","video320.m4v","video320.flv")
image_bases = []
video_bases = []
for f in existing:
if f in image_file_names:
image_bases.append(AssetFile(self,join(bases_directory,f),thumbnail_directory=thumbnail_directory,path="%s/%s" % (path,f)))
if f in video_file_names:
video_bases.append(AssetFile(self,join(bases_directory,f),thumbnail_directory=thumbnail_directory,path="%s/%s" % (path,f)))
return mimetype, category, supported, bases_directory, thumbnail_directory, path, image_bases, video_bases
def apply_single_basedir(self,repo_dir,basedir):
self.REPO_DIR = repo_dir
self.CONF_DIR = join(basedir, "conf")
self.PYTHON_DIR = join(basedir,"python")
self.JS_DIR = join(basedir,"js")
self.CSS_DIR = join(basedir,"css")
self.TEMPLATES_DIR = join(basedir,"templates")
self.SAMPLE_TEMPLATES_DIR = join(self.PYTHON_DIR,"theapps","samples","templates")
self.LIB_DIR = join(self.REPO_DIR,"python")
self.PROJECT_NAME = split(self.REPO_DIR)[1]
self.PROJECT_DIR = self.REPO_DIR
self.SOURCE_DIR = self.REPO_DIR
self.TARGET_DIR = join(self.PROJECT_DIR,"target")
self.RELEASE_DIR = join(self.PROJECT_DIR,"release")
self.UPLOADS_DIR = join("/var","uploads",self.PROJECT_NAME)
self.DOWNLOADS_DIR = join("/var","downloads",self.PROJECT_NAME)
self.BACKUPS_DIR = join("/var","backups",self.PROJECT_NAME)
self.LOG_DIR = join("/var","log",self.PROJECT_NAME)
self.SITES_DIR = dirname(self.PROJECT_DIR)
self.WEBSITE_DIRS = fs.filterdirs((
join(basedir,"root"),
join(basedir,"website"),
join(self.TARGET_DIR,"website"),
))
self.MEDIASITE_DIRS = fs.filterdirs((
join(basedir,"root"),
join(basedir,"mediasite"),
join(self.TARGET_DIR,"mediasite"),
))
self.TOOLS_DIR = expanduser("~/Sites/tools")
self.PID_DIR = join("/var/run",self.PROJECT_NAME)
from fs import filters
dirs = fs.listdir(self.PROJECT_DIR, filters=(filters.fnmatch("*site"),filters.only_directories,))
dirs.sort() # just to make sure
self.SITE_DIRS = dirs
def getSource(self, file_name):
from fs.dependency import changed_since, newer_group
src = join(structure.CSS_DIR,file_name)
if not isdir(src):
raise tornado.web.HTTPError(404)
target = join(self.application.site["target_path"],"css",file_name)
text = None
srcs = []
srcs = fs.listdir(structure.CSS_DIR,recursed=True,full_path=True)
#TODO record dependency with target
if newer_group(srcs,target) or self.request.headers.get("force",False):
lines = combine_asset_sources(src,structure.CSS_DIR,source_node=CssSourceNode)
import codecs
with codecs.open(target,"w",encoding="utf-8") as f:
text = u''.join(lines)
f.write(text)
f.flush()
if not text:
with open(target,"r") as f:
text = f.read()
return text
import fs
fs.init('myfs')
# if __name__ == '__main__':
fs.init('myfs')
fs.getcwd() # /
fs.mkdir('a0')
fs.mkdir('a0/a1')
fs.mkdir('a0/a1/a2')
fs.listdir()
fs.listdir('a0')
fs.mkdir('a0/b1')
fs.listdir('a0')
fs.chdir('a0/b1') # /a0/b1
fs.create('../a1/a2.txt', 10)
fs.listdir('../a1')
fs.chdir('/a0/a1') # /a0/a1
fs.getcwd()
try:
fs.mkdir('../a1')
except Exception, e:
print e
try:
fs.chdir('../c1')
except Exception, e:
print e
fs.getcwd()
try:
fs.deldir('/a0/a1')
except Exception, e:
print e
def populate(expander_writer,config):
if site.SCSS_DIR:
try:
import scss
setattr(scss,"LOAD_PATHS",site.SCSS_DIR)
except ImportError:
pass
""" TODO:
LOAD_PATHS = os.path.join(PROJECT_ROOT, 'sass/frameworks/')
# Assets path, where new sprite files are created:
STATIC_ROOT = os.path.join(PROJECT_ROOT, 'static/')
# Assets path, where new sprite files are created:
ASSETS_ROOT = os.path.join(PROJECT_ROOT, 'static/assets/')
# Urls for the static and assets:
STATIC_URL = '/static/'
ASSETS_URL = '/static/assets/'
"""
base_filters = [config.exclude_filter(),filters.no_hidden,filters.no_system]
css_prefix = "css"
js_prefix = "js"
if config["assets-base"]:
css_prefix = "%s/css" % config["assets-base"]
js_prefix = "%s/js" % config["assets-base"]
if site.SCSS_DIR:
for relpath in listdir(site.SCSS_DIR,filters=base_filters+[filters.fnmatch("*.scss"),]):
expander = FileExpander(site.SCSS_DIR,relpath,config=config,prefix=css_prefix)
#setattr(scss,"LOAD_PATHS",site.SCSS_DIR)
for browser in browsers:
if not config["browser"] or config["browser"] == browser.browser_type:
expander_writer(expander,browser,config)
logging.info("Cached %s for %s as %s" % (relpath,expander.domain,repr(expander)))
if site.LIBS_DIR:
builders = {}
for relpath in listdir(site.LIBS_DIR, filters=[filters.only_directories,filters.fnmatch("*.js")]):
libname = relpath[:-3]
builder = LibBuilder(site.LIBS_DIR,subdir=libname,config=config)
builders[libname] = builder
#TODO figure out the relevant one
for relpath in listdir(site.LIBS_DIR, filters=[filters.no_directories,filters.fnmatch("*.js")]):
expander = FileExpander(site.LIBS_DIR,relpath,config=config,prefix=js_prefix)
prefix = expander.name_parts[0]
if prefix in builders:
builder = builders[prefix]
else:
builder = LibBuilder(site.LIBS_DIR,prefix=prefix,config=config)
builders[prefix] = builder
builder.addVersion(expander)
for name in builders.iterkeys():
builder = builders[name]
for expander in builder.deployed_expanders:
for browser in browsers:
if not config["browser"] or config["browser"] == browser.browser_type:
#print >>sys.stderr, repr(expander)
expander_writer(expander,browser,config)
for relpath in listdir(site.SITE_DIR,recursed=True,filters=base_filters):
expander = FileExpander(site.SITE_DIR,relpath,config=config)
if relpath[0] != "_":
for browser in browsers:
if not config["browser"] or config["browser"] == browser.browser_type:
expander_writer(expander,browser,config)
logging.info("Cached %s for %s as %s" % (relpath,expander.domain,repr(expander)))
def getCollectionContents(self, respath):
return fs.listdir(respath)
def load_seed():
from thepian.conf import structure
base = join(structure.PROJECT_DIR,"seed")
if exists(base):
for a in listdir(base):
account = join(base,a)
for p in listdir(account):
project = join(account,p)
REDIS.sadd(PROJECTS_KEY % a, p)
upload_info = {
"account": a,
"project": p,
"expires": ONE_YEAR_IN_SECONDS,
}
upload_hash = hashlib.sha256( UPLOAD_SCRIPT_HASH_COMBINE % (a,p) ).hexdigest()
REDIS[UPLOAD_PATH_KEY % (p,upload_hash)] = json.dumps(upload_info)
REDIS[UPLOAD_SCRIPT_KEY % (p,upload_hash)] = load_expand_and_translate(join(structure.JS_DIR,'upload-specs.js'),**upload_info)[2]
REDIS.sadd(SHORTCUT_UPLOADS_KEY % (a,p), upload_hash)
print 'Upload URL /%s/%s/upload-specs.js' % (p,upload_hash)
suite_info = {
"account": a,
"project": p,
"suite": "all",
"exec_name": "selftest",
}
suite_src = SUITE_HASH_COMBINE % (a,p,'all','selftest')
suite_hash = hashlib.sha256(suite_src).hexdigest()
suite_id = SUITE_KEY % (p,suite_hash)
REDIS.set(suite_id , json.dumps(suite_info))
REDIS.sadd(SHORTCUT_SUITES_KEY % (a,p), suite_hash)
print 'Project %s URL /%s/%s/introduction.html' % (p,p,suite_hash)
for s in listdir(project, filters=(filters.fnmatch("*.spec.js"),)):
src,expanded,translated = load_expand_and_translate(join(project,s))
h = hashlib.sha256(src).hexdigest()
idx = 0
for spec in expanded:
spec_hash = h + "-" + str(idx)
translated = translate([ spec ])
id = TRANSLATED_SPEC_KEY % (a,p,spec_hash)
# print "adding", id, ":", t, "name", spec[2]
REDIS.set(id,translated)
info_id = SPEC_INFO_KEY % (a,p,spec_hash)
info = {"id":spec_hash,"name":spec[2],"filename":s}
REDIS.set(info_id , json.dumps(info))
examples_id = EXAMPLE_NAMES_KEY % (a,p,spec_hash)
examples = [e[1].replace('"','') for e in extract_examples(expanded)]
# print h, "seed examples", examples
# sadd is supposed to be callable by REDIS.sadd(examples_id,*examples)
for e in examples:
REDIS.sadd(examples_id, e)
# added the hash to the all list
REDIS.sadd(ALL_SPECS_KEY % (a,p), spec_hash)
++idx
print "Done loading seed."
def source_paths():
"""Iterate url paths for css and js sources"""
for name in fs.listdir(structure.CSS_DIR, filters=(only_directories,fnmatch("*.css"))):
yield join("css",name)
for name in fs.listdir(strucure.JS_DIR, filters=(only_directories,fnmatch("*.js"))):
yield join("js",name)
def ensure_presets():
home = expanduser("~/.ffmpeg")
makedirs(home)
presets = listdir(structure.CONF_DIR,filters=(filters.no_hidden,filters.no_system,filters.fnmatch("*.ffpreset")))
for preset in presets:
copy_file(src=join(structure.CONF_DIR,preset),dst=join(home,preset))
def prn(x):
try:
return "Dir " + x.dirName
except:
return "File " + x.fileName
# file system tree #1
#now it is on the root directory.
fs.init('abc')
fs.mkdir('a')
fs.mkdir('a/b1')
print fs.listdir('a')
#now it is on the root directory.
#fs.chdir('a')
fs.chdir('/')
fs.create('fa', 17)
print fs.listdir('.')
#on a directory
fs.chdir('a')
fs.mkdir('b2')
fs.mkdir('/a/b3')
def get_uploaded_assets(self):
from static import Static
return [Static(parent=self, file_name=file_name) for file_name in listdir(self.upload_directory)]
def list_sources(cls,src,full_path=True):
return fs.listdir(src,full_path=True,recursed=False,filters=(fnmatch("*.js"),))
def get_uploaded_assets(self):
from base import Asset
return [Asset(parent=self,encoded=e) for e in listdir(self.upload_directory)]
def list_dependencies(cls,src,full_path=True):
"""Naive implementation returning all files in the directory"""
return fs.listdir(src,full_path=full_path,recursed=True,followlinks=True)