def test_async_iteration_in_templates():
t = Template('{% for x in rng %}{{ x }}{% endfor %}',
enable_async=True)
async def async_iterator():
for item in [1, 2, 3]:
yield item
rv = list(t.generate(rng=async_iterator()))
assert rv == ['1', '2', '3']
def test_async_iteration_in_templates_extended():
t = Template('{% for x in rng %}{{ loop.index0 }}/{{ x }}{% endfor %}',
enable_async=True)
async def async_iterator():
for item in [1, 2, 3]:
yield item
rv = list(t.generate(rng=async_iterator()))
assert rv == ['0/1', '1/2', '2/3']
def test_async_iteration_in_templates():
t = Template("{% for x in rng %}{{ x }}{% endfor %}", enable_async=True)
async def async_iterator():
for item in [1, 2, 3]:
yield item
rv = list(t.generate(rng=async_iterator()))
assert rv == ["1", "2", "3"]
def test_async_iteration_in_templates_extended():
t = Template('{% for x in rng %}{{ loop.index0 }}/{{ x }}{% endfor %}',
enable_async=True)
async def async_iterator():
for item in [1, 2, 3]:
yield item
rv = list(t.generate(rng=async_iterator()))
assert rv == ['0/1', '1/2', '2/3']
def test_async_generate():
t = Template('{% for x in [1, 2, 3] %}{{ x }}{% endfor %}',
enable_async=True)
rv = list(t.generate())
assert rv == ['1', '2', '3']
def test_async_generate():
t = Template("{% for x in [1, 2, 3] %}{{ x }}{% endfor %}",
enable_async=True)
rv = list(t.generate())
assert rv == ["1", "2", "3"]
def test_async_iteration_in_templates_extended():
t = Template("{% for x in rng %}{{ loop.index0 }}/{{ x }}{% endfor %}",
enable_async=True)
stream = t.generate(rng=auto_aiter(range(1, 4)))
assert next(stream) == "0"
assert "".join(stream) == "/11/22/3"
class Task(BaseTask):
def __init__(self,
task_id,
command,
init_args=None,
retry_args=None,
tags=None,
units=None,
retries=1,
project_id=None,
wait_time=None):
"""
id - String. identifies the task.
command - String. script name or jinja2 template string.
init_args - List of strings. Arguments and options to add to the command.
retry_args - List of strings. If given and job is retries, use this list of arguments instead the ones
specified in init_args.
tags - List of strings. tags to add to the scheduled job.
units - Int. units to use for this scheduled job.
retries - Int. Number of retries in case job failed.
project_id - Int. Run task in given project. If not given, just run in the actual project.
wait_time - Int. Don't run the task before the given number of seconds after workflow started.
"""
super(Task, self).__init__(task_id, tags, units, retries, project_id,
wait_time)
self.command = command
self.init_args = init_args or []
self.retry_args = retry_args or []
self.__template = Template(self.command)
def as_jobgraph_dict(self):
jdict = super(Task, self).as_jobgraph_dict()
jdict.update({
'command': self.get_commands(),
'init_args': self.init_args,
'retry_args': self.retry_args,
})
return jdict
def get_commands(self):
return list(self.__template.generate())
def get_command(self, index=None):
index = index or 0
return shlex.split(self.get_commands()[index])
def get_parallel_jobs(self):
"""
Returns total number of parallel jobs that this task will consist on.
"""
return len(self.get_commands())
def get_scheduled_jobs(self, manager=None, level=0):
"""
- if level == 0, just return the task job ids (this is a second way to access self.job_ids)
- if level == 1, return the job ids of the jobs scheduled by this task jobs (typically, a crawl
manager, so returned job ids are the ids of the spider jobs scheduled by it). In this case, a
second parameter, the manager instance that schedules this task, must be provided.
"""
job_ids = super(Task, self).get_scheduled_jobs()
if level == 0:
return job_ids
assert level == 1, "Invalid level"
return [j[2] for j in get_scheduled_jobs_specs(manager, job_ids)]
def run(self, manager, retries=False, index=None):
command = self.get_command(index)
self.start_callback(manager, retries)
if index is None:
jobname = f"{manager.name}/{self.task_id}"
else:
jobname = f"{manager.name}/{self.task_id}_{index}"
if retries:
logger.info('Will retry task "%s"', jobname)
else:
logger.info('Will start task "%s"', jobname)
if retries:
cmd = command + self.retry_args
else:
cmd = command + self.init_args
jobid = manager.schedule_script(cmd,
tags=self.tags,
units=self.units,
project_id=self.project_id)
if jobid:
logger.info('Scheduled task "%s" (%s)', jobname, jobid)
self.append_jobid(jobid)
return jobid
def test_async_generate():
t = Template('{% for x in [1, 2, 3] %}{{ x }}{% endfor %}',
enable_async=True)
rv = list(t.generate())
assert rv == ['1', '2', '3']
from jinja2 import Template
from config import config
if __name__ == "__main__":
with open('template.jinja') as f:
template = Template(f.read())
with open('out.html', 'w') as f:
for l in template.generate(**config):
f.write(l)
from jinja2 import Environment, Template, select_autoescape
class sstring:
def __init__(self, sstring):
self.sstring = sstring
def superscriptify(text):
return "<sup>" + str(text.sstring) + "</sup>"
env = Environment(autoescape=select_autoescape(['html', 'xml']))
x = sstring("abc")
print(x.superscriptify())
tt = ' part1={{ var1.superscriptify() }} part2={{var2}} part3={{var3}} tail\n'
jt = Template(tt)
vars = {'var1': sstring("x1"), 'var2': "y2", "var3": "z3"}
gen = jt.generate(vars)
for x in gen:
print(pformat(x))
print("\n---\n")
print(pformat("the end"))
class Controller(object):
"""
The abstract base class for the controller of storage. This class does not actually impliment any
functionality (other than the constructor) but rather sets the method contracts for the methods supported by all
controllers.
:param template: The template string or Template instance for the jinja2 template used to create the \
contents of the tiny
:type template: str or jinja2.Template
:param str prefix_separator: The seperator between the prefix and tiny_text
:param int initial_tiny_length: The length to try for autogenerating tiny text
:param int max_retries: The max number if attempts to generate a unique tiny
:param bool overwrite: Overwrite (if tiny text is provided)
:ivar jinja2.Template template: The template used to generate the HTML of the tinyurl
:ivar str prefix_separator: The separator betweek the prefix and tiny_text for any URL created by :func:`create_url`
:ivar int initial_tiny_length: The initial length to attempt the auto_generated tiny_text, increased ny one for \
each collision after the second.
:ivar int max_retries: The maximum number of collisions that will be tolerated before raising an exception
:ivar bool overwrite: Whether a duplicat (provided) tiny_text will overwrite \
(otherwise raises :class:`TinyURLExistsException`)
"""
__metaclass__ = ABCMeta
def __init__(self, template, prefix_separator=".", tiny_length=4, max_retries=5, overwrite=False, analytics=None):
self.prefix_separator = prefix_separator
self.initial_tiny_length = tiny_length
self.max_retries = max_retries
self.overwrite = overwrite
self._template = None
self.template = template
self.analytics = analytics
@property
def template(self):
"""
The jinja2 template used to create the meta redirect
:setter: Takes either a string (which is a valid Template) or jinja2.Template object
:getter: Returns the jinja2.Template object
:type: jinja2.Template
"""
return self._template
@template.setter
def template(self, template):
"""
Set the template
:param template: The template the controller will use to generate the tinyURL
:type template: str, jinja2.Template
:raises; AttributeError
"""
if isinstance(template, Template):
self._template = template
else:
self.template = Template(template)
@abstractmethod
def put(self, url):
"""
Put the tiny url to storage.
If the tiny_text is provided in the url, it will be used (and overwritten is specified)
Otherwise, it will attempt to generate non-conflicting tiny text up to max_retries times.
Often, with large number of tiny_urls we see failure counts increase, in which case the initial_tiny_length
should be increased to increase the namespace
:param url: the stiny.utl.StaticURL to be generated
:raises: TooManyNameCollisions - if we're unable to autogenerate a tiny_text name
:raises: TinyURLExistsException - if the provided tiny exists and we're not overwriting
"""
pass
@abstractmethod
def get(self, tiny_uri):
"""
Get the tiny url from storage.
If the tiny_text is provided in the url, it will be used (and overwritten is specified)
Otherwise, it will attempt to generate non-conflicting tiny text up to max_retries times.
Often, with large number of tiny_urls we see failure counts increase, in which case the initial_tiny_length
should be increased to increase the namespace
:param tiny_uri: the name of the tiny url
:raises: TinyURLDoesNotExistsException - if the provided tiny exists and we're not overwriting
"""
pass
@abstractmethod
def delete(self, url):
"""
Delete the tiny url from storage.
:param url: the stiny.utl.StaticURL to be deleted (tiny_text must be provided) or str of tiny_text
:raises: TinyURLDoesNotExistsException - if the provided tiny does not exist
"""
pass
@abstractmethod
def list(self):
"""
List the tiny urls in storage (should not list non-tinys) in the object store
:return: Generator of stiny.url.URLs
"""
for x in xrange(10):
yield (x, x)
@abstractmethod
def exists(self, url):
"""
Check to see if the tiny_url specified exists
:param url: the stiny.utl.StaticURL to be checked (tiny_text must be provided) or str of tiny_text
:return: Boolean of existence
"""
pass
@abstractmethod
def validate(self, url):
"""
Check to see if the tiny_url specified exists and points to the correct href
:param url: the stiny.utl.StaticURL to validate
:return: Boolean of validity, nonexistance is considered nonvalid.
"""
pass
def create_url(self, *args, **kwargs):
"""
A helper function used to create URL objects that will automaticaly pass in any persistant values set on the
controller (like prefix_separator).
See :py:class:`stiny.url.URL` documentation for available parameters.
:raises: stiny.exception.InvalidURLException if the url is invalid
:return: stiny.url.URL object
"""
if 'prefix_seperator' not in kwargs:
kwargs['prefix_separator'] = self.prefix_separator
return URL(*args, **kwargs)
def _select_tiny_text(self, url):
"""
This method selects the tiny_text for the url by one of a number of methods. If the tiny text was provided \
with the URL, it uses that and errors if it exists and overwrite it not set. Otherwise, it randomly generates \
the tiny_text.
"""
if not url.tiny_text_provided:
retries = 0
while url.tiny_text is None and retries < self.max_retries:
url.tiny_text = url.generate_tiny_text(self.initial_tiny_length)
if self.exists(url.get_tiny_uri()):
retries += 1
url.tiny_text = None
if url.tiny_text is None:
raise UnableToAutogenerateTinyText(
"Unable to generate tiny name, increase retries or increase key start length or provide a name")
else:
if not self.overwrite and self.exists(url.get_tiny_uri()):
raise TinyURLExistsException("{} already exists".format(url.get_tiny_uri()))
def _get_contents_fp(self, url):
"""
Returns a file pointer to the contents of the file using StringIO.
:param url: The url to be encoded
:type url: stiny.url.URL
"""
contents = self._get_contents(url)
sio = StringIO(contents)
return sio
def _get_contents(self, url):
return "\n".join([line for line in self.template.generate(url=url, analytics=self.analytics)])
from jinja2 import Template
template_str = 'first={{first}}\n' \
'last={{last}}'
template = Template(template_str)
stream = template.stream({'first': 'John', 'last': 'Doe'})
count = 0
while True:
try:
count += 1
resolved = stream.next()
print count, resolved
except StopIteration:
break
generator = template.generate({'first': 'John', 'last': 'Doe'})
count = 0
for s in generator:
count += 1
print count, s
