def _setup(): global backup_conf_defaults backup_conf_defaults = { 'samplemethod' : conf.default_samplemethod(), 'maxrows': conf.default_maxrows(), 'samplefraction': conf.default_samplefraction() }
def __init__(self, output_var, samplemethod=None, maxrows=None, samplefraction=None, spark_events=None, coerce=None): super(SparkStoreCommand, self).__init__("", spark_events) if samplemethod is None: samplemethod = conf.default_samplemethod() if maxrows is None: maxrows = conf.default_maxrows() if samplefraction is None: samplefraction = conf.default_samplefraction() if samplemethod not in {u'take', u'sample'}: raise BadUserDataException( u'samplemethod (-m) must be one of (take, sample)') if not isinstance(maxrows, int): raise BadUserDataException(u'maxrows (-n) must be an integer') if not 0.0 <= samplefraction <= 1.0: raise BadUserDataException( u'samplefraction (-r) must be a float between 0.0 and 1.0') self.samplemethod = samplemethod self.maxrows = maxrows self.samplefraction = samplefraction self.output_var = output_var if spark_events is None: spark_events = SparkEvents() self._spark_events = spark_events self._coerce = coerce
def __init__(self, query, samplemethod=None, maxrows=None, samplefraction=None, spark_events=None): super(SQLQuery, self).__init__() if samplemethod is None: samplemethod = conf.default_samplemethod() if maxrows is None: maxrows = conf.default_maxrows() if samplefraction is None: samplefraction = conf.default_samplefraction() if samplemethod not in {u'take', u'sample'}: raise BadUserDataException( u'samplemethod (-m) must be one of (take, sample)') if not isinstance(maxrows, int): raise BadUserDataException(u'maxrows (-n) must be an integer') if not 0.0 <= samplefraction <= 1.0: raise BadUserDataException( u'samplefraction (-r) must be a float between 0.0 and 1.0') self.query = query self.samplemethod = samplemethod self.maxrows = maxrows self.samplefraction = samplefraction if spark_events is None: spark_events = SparkEvents() self._spark_events = spark_events
def __init__(self, query, samplemethod=None, maxrows=None, samplefraction=None, spark_events=None): super(SQLQuery, self).__init__() if samplemethod is None: samplemethod = conf.default_samplemethod() if maxrows is None: maxrows = conf.default_maxrows() if samplefraction is None: samplefraction = conf.default_samplefraction() if samplemethod not in {u'take', u'sample'}: raise BadUserDataException(u'samplemethod (-m) must be one of (take, sample)') if not isinstance(maxrows, int): raise BadUserDataException(u'maxrows (-n) must be an integer') if not 0.0 <= samplefraction <= 1.0: raise BadUserDataException(u'samplefraction (-r) must be a float between 0.0 and 1.0') self.query = query self.samplemethod = samplemethod self.maxrows = maxrows self.samplefraction = samplefraction if spark_events is None: spark_events = SparkEvents() self._spark_events = spark_events