def make_db(db, amount=100):
total = 0
for chunk in chunked(enumerate(gather_documents(amount), total + 1), 10, trail=True):
with db as session:
indexed = Indexed(session)
for total, document in chunk:
if len(document):
session.add(document)
indexed.add(document)
print(total, ":", repr(document))
return total
def simultaneously (function, iterable, max_workers=4) :
''' This runs the given function over the iterable concurrently, in a similar fashion to the built-in <map>
function. The output's order is not guaranteed to correspond the order of the input iterable. Therefor the
output order should be treated as undefined. The <max_workers> argument denotes the amount of worker threads to
use. '''
if max_workers < 1 :
raise ValueError('<simultaneously> requires at least one worker thread.')
with ThreadPoolExecutor(max_workers=max_workers) as executor :
futures = (executor.submit(function, item)
for item in iterable)
for chunk in chunked(futures, max_workers, trail=True) :
for future in as_completed(chunk) :
yield future.result()
def plot(
iterable, key=lambda item: item, sample_size=1, depth=1, colors=_plot_colors, basecase=_plot_basecase, _show=True
):
""" This function can be used for plotting an iterable graphically. If matplotlib isn't installed, this will only
throw a warning, and no GUI will be launched.. """
try:
pyplot
except NameError:
warnings.warn("matplotlib must be installed in order to see the plot graphic.", Warning)
else:
for i, iterable in enumerate(flattened(iterable, basecase)):
xs, ys = zip(
*(
(i, math.avg(key(item) for item in sample))
for i, sample in enumerate(iterate.chunked(iterable, sample_size, trail=True))
)
)
pyplot.plot(xs, ys, linewidth=1.0, color=colors[i % len(colors)])
if _show:
pyplot.show()
def matching (self, strings, cls=Seq, chunk_size=100) :
for chunked_strings in chunked(strings, chunk_size, trail=True) :
for match in self.session._sqlalchemy_session.query(cls).filter(cls.string.in_(chunked_strings)).all() :
yield match