def __iter__(self): # Generate the cache under any of the following conditions: # # 1. Cache file does not exist # 2. Cache file exists, is older than compile start time, and we are # forced to do so debug(D_DATASET, 'Iterating on Dataset {0}'.format(self)) if os.path.exists(self.cache_path): # If cache file is made after we started compiling, then it is # valid, so don't bother generating. if CurrentScript().start_time <= os.stat(self.cache_path).st_ctime: debug(D_DATASET, 'Loading Dataset {0}'.format(self)) return (MakeFile(f.strip(), self.nest) \ for f in open(self.cache_path, 'r')) message = 'Cache file {0} already exists'.format(self.cache_path) if CurrentScript().force: warn(D_DATASET, message) else: fatal(D_DATASET, message) debug(D_DATASET, 'Generating Dataset {0}'.format(self)) return self._generate()
from weaver.compat import map from weaver.data import MakeFile from weaver.logger import D_DATASET, debug, warn, fatal from weaver.stack import CurrentNest, CurrentScript from weaver.util import Cloneable, flatten, normalize_path, type_str import fnmatch import functools import glob import os try: from MySQLdb import connect as MySQLConnect from MySQLdb.cursors import SSDictCursor as MySQLSSDictCursor except ImportError as e: warn(D_DATASET, 'Unable to import MySQL: {0}'.format(e)) # Base Dataset class class Dataset(object): """ Weaver abstract Dataset class. In Weaver, a :class:`Dataset` is an :func:`~weaver.util.iterable` collection of :class:`weaver.data.File` objects. Each :class:`Dataset` object has a *cursor* field ``c`` which can be used to construct a query on the :class:`Dataset` object. """ def __init__(self, cache_path=None, cursor=None): self.c = cursor or ObjectCursor()