def test_input_stream(self):
task.push(FakeTask())
input_stream = datasources.input_stream_for(None, 0, 'repo://dir1/doc1.txt', None)
eq_('Hi mom!', input_stream.next().read())
task.pop()
def log_if_error(stream):
try:
count = 0
try:
for record in stream:
yield record
count += 1
except Exception, e:
print "Error {} encountered at record {} in {} {}".format(
e, count, stream, url)
finally:
task.pop()
def map_input_stream(stream, size, url, params):
from disco.util import schemesplit
import disco.func
from triv.io import datasources, task
datasources.load()
task.push(Task)
input_stream = datasources.input_stream_for(stream, size, url, params)
if input_stream:
# Note: Task is a global set by disco, we push it onto the context stap
# which will allow it to be imported by the modules that need it
return input_stream
else:
# we don't handle the given url, see if vanilla disco moduels can
task.pop() # this is normally cleared when we're done iterating
return disco.func.map_input_stream(stream,size,url,params)
def log_if_error(stream):
try:
count = 0
try:
for record in stream:
yield record
count += 1
except Exception, e:
print "Error {} encountered at record {} in {} {}".format(
e,
count,
stream,
url
)
finally:
task.pop()
def map_input_stream(stream, size, url, params):
from disco.util import schemesplit
import disco.func
from triv.io import datasources, task
datasources.load()
task.push(Task)
input_stream = datasources.input_stream_for(stream, size, url, params)
if input_stream:
# Note: Task is a global set by disco, we push it onto the context stap
# which will allow it to be imported by the modules that need it
return input_stream
else:
# we don't handle the given url, see if vanilla disco moduels can
task.pop() # this is normally cleared when we're done iterating
return disco.func.map_input_stream(stream, size, url, params)
