def _setup_template_context(cls):
'''Setup Pylons global tmpl_context (c)
'''
c1 = ContextObj()
pylons_context = PylonsContext()
pylons_context.tmpl_context = c1
pylons.tmpl_context._push_object(c1)
def setup_app_env(self, environ, start_response):
"""Setup and register all the Pylons objects with the registry
After creating all the global objects for use in the request,
:meth:`~PylonsApp.register_globals` is called to register them
in the environment.
"""
if self.log_debug:
log.debug("Setting up Pylons stacked object globals")
# Setup the basic pylons global objects
req_options = self.request_options
req = Request(environ,
charset=req_options['charset'],
unicode_errors=req_options['errors'],
decode_param_names=req_options['decode_param_names'])
req.language = req_options['language']
req.config = self.config
req.link, req.route_dict = environ['wsgiorg.routing_args']
response = Response(content_type=self.response_options['content_type'],
charset=self.response_options['charset'])
response.headers.update(self.response_options['headers'])
# Store a copy of the request/response in environ for faster access
pylons_obj = PylonsContext()
pylons_obj.config = self.config
pylons_obj.request = req
pylons_obj.response = response
pylons_obj.app_globals = self.globals
pylons_obj.h = self.helpers
if 'routes.url' in environ:
pylons_obj.url = environ['routes.url']
environ['pylons.pylons'] = pylons_obj
environ['pylons.environ_config'] = self.environ_config
# Setup the translator object
lang = self.config['lang']
pylons_obj.translator = _get_translator(lang,
pylons_config=self.config)
if self.config['pylons.strict_tmpl_context']:
tmpl_context = ContextObj()
else:
tmpl_context = AttribSafeContextObj()
pylons_obj.tmpl_context = req.tmpl_context = tmpl_context
if self._session_key in environ:
pylons_obj.session = req.session = environ[self._session_key]
if self._cache_key in environ:
pylons_obj.cache = environ[self._cache_key]
# Load the globals with the registry if around
if 'paste.registry' in environ:
self.register_globals(environ)
def setup_app_env(self, environ, start_response):
"""Setup and register all the Pylons objects with the registry
After creating all the global objects for use in the request,
:meth:`~PylonsApp.register_globals` is called to register them
in the environment.
"""
if self.log_debug:
log.debug("Setting up Pylons stacked object globals")
# Setup the basic pylons global objects
req_options = self.request_options
req = Request(environ, charset=req_options['charset'],
unicode_errors=req_options['errors'],
decode_param_names=req_options['decode_param_names'])
req.language = req_options['language']
response = Response(
content_type=self.response_options['content_type'],
charset=self.response_options['charset'])
response.headers.update(self.response_options['headers'])
# Store a copy of the request/response in environ for faster access
pylons_obj = PylonsContext()
pylons_obj.config = self.config
pylons_obj.request = req
pylons_obj.response = response
pylons_obj.g = pylons_obj.app_globals = self.globals
pylons_obj.h = self.helpers
if self.buffet:
pylons_obj.buffet = self.buffet
environ['pylons.pylons'] = pylons_obj
environ['pylons.environ_config'] = self.environ_config
# Setup the translator object
lang = self.config['lang']
pylons_obj.translator = _get_translator(lang,
pylons_config=self.config)
if self.config['pylons.strict_c']:
c = ContextObj()
else:
c = AttribSafeContextObj()
pylons_obj.c = c
econf = self.config['pylons.environ_config']
if self._session_key in environ:
pylons_obj.session = environ[self._session_key]
if self._cache_key in environ:
pylons_obj.cache = environ[self._cache_key]
# Load the globals with the registry if around
if 'paste.registry' in environ:
self.register_globals(environ)
def setUp(self):
c = ContextObj()
py_obj = PylonsContext()
py_obj.c = c
py_obj.request = py_obj.response = None
self.environ = {'pylons.routes_dict':dict(action='index'),
'paste.config':dict(global_conf=dict(debug=True)),
'pylons.pylons':py_obj}
pylons.c._push_object(c)
def setup_class(cls):
"""Set up tests."""
cls.some_data_dict = {'sort': u'metadata_modified desc',
'fq': '',
'rows': 20,
'facet.field': ['groups',
'tags',
'extras_fformat',
'license',
'authorstring',
'organizationstring',
'extras_language'],
'q': u'',
'start': 0,
'extras': {'ext_author-4': u'testauthor',
'ext_date-metadata_modified-end': u'2013',
'ext_date-metadata_modified-start': u'2000',
'ext_groups-6': u'testdiscipline',
'ext_operator-2': u'OR',
'ext_operator-3': u'AND',
'ext_operator-4': u'AND',
'ext_operator-5': u'OR',
'ext_operator-6': u'NOT',
'ext_organization-3': u'testorg',
'ext_tags-1': u'testkeywd',
'ext_tags-2': u'testkeywd2',
'ext_title-5': u'testtitle'}
}
cls.short_data_dict = {'sort': u'metadata_modified desc',
'fq': '',
'rows': 20,
'facet.field': ['groups',
'tags',
'extras_fformat',
'license',
'authorstring',
'organizationstring',
'extras_language'],
'q': u'',
'start': 0,
}
cls.test_q_terms = u' ((tags:testkeywd) OR ( tags:testkeywd2 AND ' + \
u'organization:testorg AND author:testauthor) OR ' + \
u'( title:testtitle NOT groups:testdiscipline))'
cls.test_q_dates = u' metadata_modified:[2000-01-01T00:00:00.000Z TO ' + \
u'2013-12-31T23:59:59.999Z]'
cls.kata_plugin = KataPlugin()
# The Pylons globals are not available outside a request. This is a hack to provide context object.
c = AttribSafeContextObj()
py_obj = PylonsContext()
py_obj.tmpl_context = c
pylons.tmpl_context._push_object(c)
def setUp(self):
c = ContextObj()
py_obj = PylonsContext()
py_obj.tmpl_context = c
py_obj.request = py_obj.response = None
self.environ = {
'pylons.routes_dict': dict(action='index'),
'paste.config': dict(global_conf=dict(debug=True)),
'pylons.pylons': py_obj
}
pylons.tmpl_context._push_object(c)
def setUp(self):
if not os.path.exists(session_dir):
os.makedirs(session_dir)
# Setting TG2 up:
c = ContextObj()
py_obj = PylonsContext()
py_obj.c = c
py_obj.request = py_obj.response = None
environ = {'pylons.routes_dict': dict(action='index'),
'pylons.pylons': py_obj}
pylons.tmpl_context._push_object(c)
self.app = make_app(DefaultLessTGController, environ, with_errors=True)
def setUp(self):
import pylons
from pylons.util import ContextObj, PylonsContext
c = ContextObj()
py_obj = PylonsContext()
py_obj.tmpl_context = c
py_obj.request = py_obj.response = None
self.environ = {'pylons.routes_dict':dict(action='index'),
'paste.config':dict(global_conf=dict(debug=True)),
'pylons.pylons':py_obj}
pylons.tmpl_context._push_object(c)
def setUp(self):
if not os.path.exists(session_dir):
os.makedirs(session_dir)
# Setting TG2 up:
c = ContextObj()
py_obj = PylonsContext()
py_obj.c = c
py_obj.request = py_obj.response = None
environ = {'pylons.routes_dict': dict(action='index'),
'pylons.pylons': py_obj}
pylons.tmpl_context._push_object(c)
self.app = make_app(DefaultLessTGController, environ, with_errors=True)
def setup_class(cls):
'''
Setup database
'''
model.repo.rebuild_db()
harvest_model.setup()
kata_model.setup()
# The Pylons globals are not available outside a request. This is a hack to provide context object.
c = AttribSafeContextObj()
py_obj = PylonsContext()
py_obj.tmpl_context = c
pylons.tmpl_context._push_object(c)
def setup_class(cls):
'''
Setup database
'''
model.repo.rebuild_db()
harvest_model.setup()
kata_model.setup()
# The Pylons globals are not available outside a request. This is a hack to provide context object.
c = AttribSafeContextObj()
py_obj = PylonsContext()
py_obj.tmpl_context = c
pylons.tmpl_context._push_object(c)
def setUp(self):
# Creating the session dir:
if not os.path.exists(session_dir):
os.makedirs(session_dir)
# Setting TG2 up:
c = ContextObj()
py_obj = PylonsContext()
py_obj.c = c
py_obj.request = py_obj.response = None
environ = {"pylons.routes_dict": dict(action="index"), "pylons.pylons": py_obj}
pylons.tmpl_context._push_object(c)
# Finally, the app:
self.app = make_app(self.controller, environ)
def setUp(self):
# Creating the session dir:
if not os.path.exists(session_dir):
os.makedirs(session_dir)
# Setting TG2 up:
c = ContextObj()
py_obj = PylonsContext()
py_obj.c = c
py_obj.request = py_obj.response = None
environ = {'pylons.routes_dict': dict(action='index'),
'pylons.pylons': py_obj}
pylons.tmpl_context._push_object(c)
# Finally, the app:
self.app = make_app(self.controller, environ)
def setUp(self):
# Creating the session dir:
if not os.path.exists(session_dir):
os.makedirs(session_dir)
# Setting Pylons up:
c = ContextObj()
py_obj = PylonsContext()
py_obj.c = c
py_obj.request = py_obj.response = None
self.environ = {
'pylons.routes_dict': dict(action='index'),
'paste.config': dict(global_conf=dict(debug=True)),
'pylons.pylons': py_obj,
}
pylons.c._push_object(c)
# Finally, the app:
self.app = make_app(self.controller, self.environ)
def setUp(self):
# Creating the session dir:
if not os.path.exists(session_dir):
os.makedirs(session_dir)
# Setting Pylons up:
c = ContextObj()
py_obj = PylonsContext()
py_obj.c = c
py_obj.request = py_obj.response = None
self.environ = {
'pylons.routes_dict': dict(action='index'),
'paste.config': dict(global_conf=dict(debug=True)),
'pylons.pylons': py_obj,
}
pylons.c._push_object(c)
# Finally, the app:
self.app = make_app(self.controller, self.environ)
def __call__(self, environ, start_response):
registry = environ['paste.registry']
py_obj = PylonsContext()
environ_config = environ.setdefault('pylons.environ_config', {})
if self.setup_cache:
py_obj.cache = environ['beaker.cache']
registry.register(pylons.cache, environ['beaker.cache'])
environ_config['cache'] = 'beaker.cache'
if self.setup_session:
py_obj.session = environ['beaker.session']
registry.register(pylons.session, environ['beaker.session'])
environ_config['session'] = 'beaker.session'
if self.setup_g:
py_obj.g = self.g
registry.register(pylons.g, self.g)
translator = gettext.NullTranslations()
py_obj.translator = translator
registry.register(pylons.translator, translator)
# Update the environ
environ.update(self.environ)
py_obj.config = pylons.config._current_obj()
py_obj.request = req = Request(environ)
py_obj.response = resp = Response()
py_obj.c = ContextObj()
environ['pylons.pylons'] = py_obj
registry.register(pylons.request, req)
registry.register(pylons.response, resp)
resp.content_type = 'text/html'
resp.charset = 'utf-8'
return self.app(environ, start_response)
def setup_class(cls):
"""Set up testing environment."""
model.repo.rebuild_db()
kata_model.setup()
CreateTestData.create()
comments_model.init_tables()
wsgiapp = make_app(config['global_conf'], **config['app_conf'])
cls.app = paste.fixture.TestApp(wsgiapp)
# The Pylons globals are not available outside a request. This is a hack to provide context object.
c = AttribSafeContextObj()
py_obj = PylonsContext()
py_obj.tmpl_context = c
pylons.tmpl_context._push_object(c)
# The lxml's HTML parser allows quite much errors so it is not really a good validation of the rendered pages.
cls.html_parser = etree.HTMLParser()
def __call__(self, environ, start_response):
registry = environ['paste.registry']
py_obj = PylonsContext()
environ_config = environ.setdefault('pylons.environ_config', {})
if self.setup_cache:
py_obj.cache = environ['beaker.cache']
registry.register(pylons.cache, environ['beaker.cache'])
environ_config['cache'] = 'beaker.cache'
if self.setup_session:
py_obj.session = environ['beaker.session']
registry.register(pylons.session, environ['beaker.session'])
environ_config['session'] = 'beaker.session'
if self.setup_g:
py_obj.app_globals = self.g
registry.register(pylons.app_globals, self.g)
translator = gettext.NullTranslations()
py_obj.translator = translator
registry.register(pylons.translator, translator)
# Update the environ
req = Request(
environ,
charset=request_defaults['charset'],
unicode_errors=request_defaults['errors'],
decode_param_names=request_defaults['decode_param_names'])
req.language = request_defaults['language']
response = Response(content_type=response_defaults['content_type'],
charset=response_defaults['charset'])
response.headers.update(response_defaults['headers'])
environ.update(self.environ)
py_obj.config = pylons.config._current_obj()
py_obj.request = req
py_obj.response = response
py_obj.tmpl_context = ContextObj()
environ['pylons.pylons'] = py_obj
registry.register(pylons.request, req)
registry.register(pylons.response, response)
if 'routes.url' in environ:
registry.register(pylons.url, environ['routes.url'])
return self.app(environ, start_response)
def __call__(self, environ, start_response):
registry = environ['paste.registry']
py_obj = PylonsContext()
environ_config = environ.setdefault('pylons.environ_config', {})
if self.setup_cache:
py_obj.cache = environ['beaker.cache']
registry.register(pylons.cache, environ['beaker.cache'])
environ_config['cache'] = 'beaker.cache'
if self.setup_session:
py_obj.session = environ['beaker.session']
registry.register(pylons.session, environ['beaker.session'])
environ_config['session'] = 'beaker.session'
if self.setup_g:
py_obj.app_globals = self.g
registry.register(pylons.app_globals, self.g)
translator = gettext.NullTranslations()
py_obj.translator = translator
registry.register(pylons.translator, translator)
# Update the environ
req = Request(environ, charset=request_defaults['charset'],
unicode_errors=request_defaults['errors'],
decode_param_names=request_defaults['decode_param_names']
)
req.language = request_defaults['language']
response = Response(
content_type=response_defaults['content_type'],
charset=response_defaults['charset'])
response.headers.update(response_defaults['headers'])
environ.update(self.environ)
py_obj.config = pylons.config._current_obj()
py_obj.request = req
py_obj.response = response
py_obj.tmpl_context = ContextObj()
environ['pylons.pylons'] = py_obj
registry.register(pylons.request, req)
registry.register(pylons.response, response)
if 'routes.url' in environ:
registry.register(pylons.url, environ['routes.url'])
return self.app(environ, start_response)
def __init__(self, *args, **kw):
RepozeBFGRequest.__init__(self, *args, **kw)
attrs = self.__dict__
attrs['tmpl_context'] = PylonsContext()
def main(argv):
log = logging.getLogger()
logging.basicConfig(level=logging.INFO)
conf = appconfig('config:development.ini', relative_to='.')
config = None
if not pylons.test.pylonsapp:
config = load_environment(conf.global_conf, conf.local_conf)
# Create the tables if they don't already exist
#metadata.create_all(bind=Session.bind)
c = ContextObj()
py_obj = PylonsContext()
py_obj.tmpl_context = c
pylons.tmpl_context._push_object(c)
#corpushistory = model.meta.Session.query(model.Corpusversion).all()
corpusversion = model.meta.Session.query(model.Corpusversion).order_by(model.Corpusversion.updated).first()
c.iso_time = corpusversion.updated.strftime("%Y-%m-%d")
c.version_number = "{0}.{1}".format(corpusversion.version, corpusversion.revision)
# template_entries_seg
mylookup = TemplateLookup(directories=config['pylons.paths']['templates'])
template_header = open(os.path.join(config['pylons.paths']['templates'][0], 'base', 'graf-header.hdr')).read()
template_entries = open(os.path.join(config['pylons.paths']['templates'][0], 'base', 'graf-entries.txt')).read()
template_entries_seg = open(os.path.join(config['pylons.paths']['templates'][0], 'base', 'graf-entries.xml')).read()
template_annotations = open(os.path.join(config['pylons.paths']['templates'][0], 'base', 'graf-annotations.xml')).read()
#template_annotations_seg = open(os.path.join(config['pylons.paths']['templates'][0], 'base', 'graf-annotations-seg.xml')).read()
metadata_file = codecs.open(os.path.join(config['pylons.paths']['static_files'], 'downloads', "xml", "sources.csv"), "w", "utf-8")
metadata_file.write("ID\tTYPE\tLANGUAGES\tIS_READY\tTITLE\tCOMPONENT\n")
#http://www.cidles.eu/quanthistling/book/minor1987/hto/spa?format=xml
for b in quanthistling.dictdata.books.list + quanthistling.dictdata.toolboxfiles.list:
#if b['bibtex_key'] != "leach1969":
# continue
c.book = model.meta.Session.query(model.Book).filter_by(bibtex_key=b['bibtex_key']).first()
if c.book:
# escape characters for XML
c.bookinfo = escape(c.book.bookinfo())
c.book_title = escape(c.book.title)
c.book_author = escape(c.book.author)
# collect book data
languages = [ l.language_iso.langcode for dictdata in c.book.dictdata for l in dictdata.src_languages + dictdata.tgt_languages if l.language_iso]
components = [ dictdata.component.name for dictdata in c.book.dictdata ]
metadata_file.write(u"{0}\t{1}\t{2}\t{3}\t{4}\t{5}\n".format(c.book.bibtex_key, "dictionary", ",".join(languages), c.book.is_ready, c.book.bookinfo(), ",".join(components)))
print "Exporting XML data for %s..." % b['bibtex_key']
#temppath = tempfile.mkdtemp()
temppath = os.path.join(config['pylons.paths']['static_files'], 'downloads', 'xml', b['bibtex_key'])
if not os.path.exists(temppath):
os.mkdir(temppath)
else:
files = glob.glob(os.path.join(temppath, "*"))
for f in files:
os.remove(f)
for c.dictdata in c.book.dictdata:
c.url_for = url_for
c.base_url = "http://www.quanthistling.info/data"
#c.relative_url = url_for(controller='book', action='dictdata', bibtexkey=c.book.bibtex_key, startpage=c.dictdata.startpage, endpage=c.dictdata.endpage, format='html')
#c.heading = c.book.bookinfo()
c.basename = "dict-%s-%i-%i" % (b['bibtex_key'], c.dictdata.startpage, c.dictdata.endpage)
print " getting entries..."
c.entries = model.meta.Session.query(model.Entry).filter(model.Entry.dictdata_id==c.dictdata.id).order_by("startpage", "pos_on_page").all()
print " getting annotations..."
annotations = model.meta.Session.query(model.Annotation).join(model.Entry, model.Annotation.entry_id==model.Entry.id).filter(model.Entry.dictdata_id==c.dictdata.id).order_by("startpage", "pos_on_page").all()
c.annotations = collections.defaultdict(dict)
for a in annotations:
if not c.annotations[a.entry_id]:
c.annotations[a.entry_id] = collections.defaultdict(list)
c.annotations[a.entry_id][(a.start, a.end)].append(a)
print " getting counts..."
c.count_heads = model.meta.Session.query(model.Annotation).join(model.Entry, model.Annotation.entry_id==model.Entry.id).filter(model.Entry.dictdata_id==c.dictdata.id).filter(model.Annotation.value==u"head").count()
c.count_translations = model.meta.Session.query(model.Annotation).join(model.Entry, model.Annotation.entry_id==model.Entry.id).filter(model.Entry.dictdata_id==c.dictdata.id).filter(model.Annotation.value==u"translation").count()
c.count_pos = model.meta.Session.query(model.Annotation).join(model.Entry, model.Annotation.entry_id==model.Entry.id).filter(model.Entry.dictdata_id==c.dictdata.id).filter(model.Annotation.value==u"pos").count()
c.count_examples_src = model.meta.Session.query(model.Annotation).join(model.Entry, model.Annotation.entry_id==model.Entry.id).filter(model.Entry.dictdata_id==c.dictdata.id).filter(model.Annotation.value==u"example-src").count()
c.count_examples_tgt = model.meta.Session.query(model.Annotation).join(model.Entry, model.Annotation.entry_id==model.Entry.id).filter(model.Entry.dictdata_id==c.dictdata.id).filter(model.Annotation.value==u"example-tgt").count()
c.count_manually_corrected = model.meta.Session.query(model.Entry).filter(model.Entry.dictdata_id==c.dictdata.id).filter(model.Entry.has_manual_annotations==True).count()
#xml = render('/derived/book/dictdata.xml')
#xml = literal(template.render_unicode(c))
print " header..."
# write header
xml = Template(template_header, lookup=mylookup).render_unicode(c=c)
oFile = open(os.path.join(temppath, "%s.hdr" % c.basename),'wb')
oFile.write(xml.encode("utf-8"))
oFile.close()
print " base data..."
# write base data file
xml = Template(template_entries, lookup=mylookup).render_unicode(c=c)
oFile = open(os.path.join(temppath, "%s.txt" % c.basename),'wb')
oFile.write(xml.encode("utf-8"))
oFile.close()
print " entries..."
# write entry file
xml = Template(template_entries_seg, lookup=mylookup).render_unicode(c=c)
oFile = open(os.path.join(temppath, "%s-entries.xml" % c.basename),'wb')
oFile.write(xml.encode("utf-8"))
oFile.close()
print " formatting annotations..."
c.annotationtypes = [ "pagelayout", "formatting" ]
c.annotationname = "formatting"
xml = Template(template_annotations, lookup=mylookup).render_unicode(c=c)
oFile = open(os.path.join(temppath, "%s-formatting.xml" % c.basename),'wb')
oFile.write(xml.encode("utf-8"))
oFile.close()
print " dictinterpretation annotations..."
c.annotationtypes = [ "dictinterpretation", "orthographicinterpretation", "errata" ]
c.annotationname = "dictinterpretation"
xml = Template(template_annotations, lookup=mylookup).render_unicode(c=c)
oFile = open(os.path.join(temppath, "%s-dictinterpretation.xml" % c.basename),'wb')
oFile.write(xml.encode("utf-8"))
oFile.close()
# create archive
myzip = zipfile.ZipFile(os.path.join(config['pylons.paths']['static_files'], 'downloads', 'xml', '%s.zip' % b['bibtex_key']), 'w', zipfile.ZIP_DEFLATED)
for file in glob.glob(os.path.join(temppath, "*.*")):
myzip.write(file, os.path.basename(file))
myzip.close()
#shutil.rmtree(temppath)
metadata_file.close()
myzip = zipfile.ZipFile(os.path.join(config['pylons.paths']['static_files'], 'downloads', 'xml', 'data.zip'), 'w', zipfile.ZIP_DEFLATED)
graf_dirs = [d for d in glob.glob(os.path.join(config['pylons.paths']['static_files'], 'downloads', 'xml', "*")) if os.path.isdir(d)]
for d in graf_dirs:
bibtex_key = d[d.rfind(os.sep)+1:]
for f in glob.glob(os.path.join(d, "*.*")):
myzip.write(f, os.path.join(bibtex_key, os.path.basename(f)))
f = os.path.join(config['pylons.paths']['static_files'], 'downloads', 'xml', 'sources.csv')
myzip.write(f, os.path.basename(f))
myzip.close()
pylons.tmpl_context._pop_object()