def _fixed_lookup_fixture(self):
return [
(sqltypes.String(), sqltypes.VARCHAR()),
(sqltypes.String(1), sqltypes.VARCHAR(1)),
(sqltypes.String(3), sqltypes.VARCHAR(3)),
(sqltypes.Text(), sqltypes.TEXT()),
(sqltypes.Unicode(), sqltypes.VARCHAR()),
(sqltypes.Unicode(1), sqltypes.VARCHAR(1)),
(sqltypes.UnicodeText(), sqltypes.TEXT()),
(sqltypes.CHAR(3), sqltypes.CHAR(3)),
(sqltypes.NUMERIC, sqltypes.NUMERIC()),
(sqltypes.NUMERIC(10, 2), sqltypes.NUMERIC(10, 2)),
(sqltypes.Numeric, sqltypes.NUMERIC()),
(sqltypes.Numeric(10, 2), sqltypes.NUMERIC(10, 2)),
(sqltypes.DECIMAL, sqltypes.DECIMAL()),
(sqltypes.DECIMAL(10, 2), sqltypes.DECIMAL(10, 2)),
(sqltypes.INTEGER, sqltypes.INTEGER()),
(sqltypes.BIGINT, sqltypes.BIGINT()),
(sqltypes.Float, sqltypes.FLOAT()),
(sqltypes.TIMESTAMP, sqltypes.TIMESTAMP()),
(sqltypes.DATETIME, sqltypes.DATETIME()),
(sqltypes.DateTime, sqltypes.DATETIME()),
(sqltypes.DateTime(), sqltypes.DATETIME()),
(sqltypes.DATE, sqltypes.DATE()),
(sqltypes.Date, sqltypes.DATE()),
(sqltypes.TIME, sqltypes.TIME()),
(sqltypes.Time, sqltypes.TIME()),
(sqltypes.BOOLEAN, sqltypes.BOOLEAN()),
(sqltypes.Boolean, sqltypes.BOOLEAN()),
]
def _fixed_lookup_fixture(self):
return [
(sqltypes.String(), sqltypes.VARCHAR()),
(sqltypes.String(1), sqltypes.VARCHAR(1)),
(sqltypes.String(3), sqltypes.VARCHAR(3)),
(sqltypes.Text(), sqltypes.TEXT()),
(sqltypes.Unicode(), sqltypes.VARCHAR()),
(sqltypes.Unicode(1), sqltypes.VARCHAR(1)),
(sqltypes.UnicodeText(), sqltypes.TEXT()),
(sqltypes.CHAR(3), sqltypes.CHAR(3)),
(sqltypes.NUMERIC, sqltypes.NUMERIC()),
(sqltypes.NUMERIC(10, 2), sqltypes.NUMERIC(10, 2)),
(sqltypes.Numeric, sqltypes.NUMERIC()),
(sqltypes.Numeric(10, 2), sqltypes.NUMERIC(10, 2)),
(sqltypes.DECIMAL, sqltypes.DECIMAL()),
(sqltypes.DECIMAL(10, 2), sqltypes.DECIMAL(10, 2)),
(sqltypes.INTEGER, sqltypes.INTEGER()),
(sqltypes.BIGINT, sqltypes.BIGINT()),
(sqltypes.Float, sqltypes.FLOAT()),
(sqltypes.TIMESTAMP, sqltypes.TIMESTAMP()),
(sqltypes.DATETIME, sqltypes.DATETIME()),
(sqltypes.DateTime, sqltypes.DATETIME()),
(sqltypes.DateTime(), sqltypes.DATETIME()),
(sqltypes.DATE, sqltypes.DATE()),
(sqltypes.Date, sqltypes.DATE()),
(sqltypes.TIME, sqltypes.TIME()),
(sqltypes.Time, sqltypes.TIME()),
(sqltypes.BOOLEAN, sqltypes.BOOLEAN()),
(sqltypes.Boolean, sqltypes.BOOLEAN()),
(sqlite.DATE(storage_format="%(year)04d%(month)02d%(day)02d", ),
sqltypes.DATE()),
(sqlite.TIME(
storage_format="%(hour)02d%(minute)02d%(second)02d", ),
sqltypes.TIME()),
(sqlite.DATETIME(storage_format="%(year)04d%(month)02d%(day)02d"
"%(hour)02d%(minute)02d%(second)02d", ),
sqltypes.DATETIME()),
]
def stdtype_to_sqltype(stdtype):
import sqlalchemy.types as sqltypes
if isinstance(stdtype, stdtypes.StringType):
return sqltypes.VARCHAR(length=stdtype.max_len) if 0 < stdtype.max_len < 65536 else sqltypes.TEXT()
if isinstance(stdtype, stdtypes.BoolType):
return sqltypes.BOOLEAN()
if isinstance(stdtype, stdtypes.DateType):
return sqltypes.DATE() if stdtype.only_date else sqltypes.TIMESTAMP()
if isinstance(stdtype, stdtypes.IntegerType):
return sqltypes.BIGINT() if stdtype.length > 11 else sqltypes.INTEGER()
if isinstance(stdtype, stdtypes.DecimalType):
return sqltypes.DECIMAL()
if isinstance(stdtype, stdtypes.ArrayType):
return sqltypes.ARRAY(item_type=stdtype.item_type)
def get_spark_type(field, required_type):
if isinstance(required_type, type(db_types.DATE())):
return spk_types.StructField(field, spk_types.DateType(), True)
elif isinstance(required_type, type(db_types.DATETIME())):
return spk_types.StructField(field, spk_types.TimestampType(), True)
elif isinstance(required_type, type(db_types.VARCHAR())):
return spk_types.StructField(field, spk_types.StringType(), True)
elif isinstance(required_type, type(db_types.INT())):
return spk_types.StructField(
field, spk_types.LongType(), True
) # db type enforced earlier than spark ones, so spark types needs to be less restrictive than spark ones so needs to choose LongType instead of IntegerType
elif isinstance(required_type, type(db_types.FLOAT())):
return spk_types.StructField(field, spk_types.FloatType(), True)
elif isinstance(required_type, type(db_types.BOOLEAN())):
return spk_types.StructField(field, spk_types.BooleanType(), True)
else:
raise Exception(
"Type not recognized, field={}, required_type={}".format(
field, required_type))
def str_to_sqltype(expr):
import re
import sqlalchemy.types as sqltypes
norm_expr = expr.lower()
if norm_expr.startswith('integer'):
match_result = re.match(r'integer\((\d+)\)', norm_expr)
if match_result is not None:
return sqltypes.BIGINT() if int(match_result.group(1)) > 11 else sqltypes.INTEGER()
return sqltypes.BIGINT()
if norm_expr == 'decimal':
return sqltypes.DECIMAL()
if norm_expr == 'date':
return sqltypes.DATETIME()
if norm_expr == 'bool' or norm_expr == 'boolean':
return sqltypes.BOOLEAN()
if norm_expr.startswith('string'):
match_result = re.match(r'string\((\d+)\)', norm_expr)
if match_result is not None:
maxlen = int(match_result.group(1))
return sqltypes.VARCHAR(maxlen) if maxlen < 65536 else sqltypes.TEXT
return sqltypes.TEXT()
raise RuntimeError("Unsupported data type [" + expr + "]")
def boolean(**kwargs):
return _vary(types.BOOLEAN(), boolean_map.copy(), kwargs)
from sqlalchemy_solr.solr_type_compiler import SolrTypeCompiler
from sqlalchemy_solr.solrdbapi.array import ARRAY
logging.basicConfig(format="%(asctime)s - %(message)s", level=logging.ERROR)
_type_map = {
"binary": types.LargeBinary(),
"boolean": types.Boolean(),
"pdate": types.DateTime(),
"pint": types.Integer(),
"plong": types.BigInteger(),
"pfloat": types.Float(),
"pdouble": types.REAL(),
"string": types.VARCHAR(),
"text_general": types.Text(),
"booleans": ARRAY(types.BOOLEAN()),
"pints": ARRAY(types.Integer()),
"plongs": ARRAY(types.BigInteger()),
"pfloats": ARRAY(types.Float()),
"pdoubles": ARRAY(types.REAL()),
"strings": ARRAY(types.VARCHAR()),
}
class SolrCompiler(compiler.SQLCompiler):
merge_ops = (operators.ge, operators.gt, operators.le, operators.lt)
bounds = {
operators.ge: "[",
operators.gt: "{",
operators.le: "]",
