def __call__(self, context):
if isinstance(self._from_variable, types.StringTypes):
# 如果是字符串，那么将作为上下文变量名，直接从context中获取变量
obj = context[self._from_variable]
table = TableWrapper(self._name, self._titles,
self._table_type)(obj)
context[self._to_variable] = table
return table
elif isinstance(self._from_variable, SequenceCollectionType):
# 如果是List或者Tuple类型，那么从头构造数据
table_data = []
for row in self._from_variable:
if isinstance(row, SequenceCollectionType):
row_data = []
for element in row:
if isinstance(element, types.StringTypes) \
and element.startswith("$"):
element = context[element[1:]]
row_data.append(element)
table_data.append(row_data)
elif (isinstance(row, types.StringTypes)
and row.startswith("$")):
row = context[row[1:]]
table_data.append(row)
else:
table_data.append(row)
table = TableWrapper(self._name, self._titles,
self._table_type)(table_data)
context[self._to_variable] = table
return table
def setUp(self):
self.table_a = TableWrapper(
"table_a",
("id", None, "name", None, "grade", None, "gender", None))(
((1, "Sam", "A", "M"), (2, "Jack", "B", "M"), (3, "Lucy", "B",
"F")))
self.table_b = TableWrapper(
"table_b",
("id", None, "name", None, "grade", None, "gender", None))(
((4, "James", "A", "M"), (5, "Peter", "A", "M")))
self.table_c = TableWrapper(
"table_c", ("name", None, "grade", None, "gender", None))(
(("Betty", "C", "F"), ("May", "C", "F")))
def execute(self, context, table, split_condition, variable=None):
"""
:param context 上下文对象
:param table 要分割的table对象
:param split_condition 接受一个函数对象，参数为一个行对象，结果返回两个值，
第一个为引用的key，第二个为表格名称
:param variable 要保存到的上下文变量
:return 返回一个字典对象，key为split_condition中返回的引用值，value为表格对象
"""
result = {}
for row in table:
split_result = split_condition(row)
if split_result is None:
continue
ref_key, sub_table_name = split_result
if ref_key in result:
sub_table = result[ref_key]
sub_table.append(row.obj)
else:
sub_table = TableWrapper(sub_table_name,
table.titles)([row.obj])
result[ref_key] = sub_table
if variable:
context[variable] = result
return result
def test_read_line_with_file(self):
ctx = {}
# 测试一般执行
result = JSONR("line_file.json", "line")(ctx)
self.assertEquals(len(result), 3)
self.assertEquals(result[0], {"id": 1, "name": "Sam", "grade": "A"})
# 测试record_handler
JSONR("line_file.json",
"line",
record_handler=lambda row: {row.pop("name"): row},
variable="test")(ctx)
test_result = ctx["test"]
self.assertEquals(test_result[0], {"Sam": {"id": 1, "grade": "A"}})
# 测试result_wrapper
table = JSONR("line_file.json",
"line",
result_wrapper=TableWrapper(
"students", (Title("id", u"编号"), Title(
"name", u"姓名"), Title("grade", u"等级"))))(ctx)
print "\n", table
row_0 = table[0]
self.assertEquals((row_0.id, row_0.name, row_0.grade), (1, "Sam", "A"))
def test_read_json_array(self):
ctx = {}
# 测试一般执行
result = JSONR("array_file.json", "array")(ctx)
self.assertEquals(len(result), 3)
self.assertEquals(result[0], {"id": 1, "name": "Jack", "grade": "A"})
# URL加载执行
result = JSONR("http://test.gf/array", "array")(ctx)
self.assertEquals(len(result), 3)
self.assertEquals(result[0], {"id": 1, "name": "Jack", "grade": "A"})
# 测试record_handler
result = JSONR("http://test.gf/array",
"array",
record_handler=lambda r: Student(**r))(ctx)
std = result[0]
self.assertEquals((std.id, std.name, std.grade), (1, "Jack", "A"))
table = JSONR("http://test.gf/array",
"array",
record_handler=lambda r: Student(**r),
result_wrapper=TableWrapper(
"student",
(Title("id"), Title("name"), Title("grade"))))(ctx)
print "\n", table
row_0 = table[0]
self.assertEquals((row_0.id, row_0.name, row_0.grade),
(1, "Jack", "A"))
def _gen_orm_query_args(ctx):
sqltasks = ctx["task"]["sqltasks"]
return [
SQL(engine_name=t["db"],
variable_name="table_%d" % idx,
sql=t["sql"],
result_wrapper=TableWrapper(t["table"],
t["titles"],
auto_title_name=True))
for idx, t in enumerate(sqltasks)
]
def test_query(self):
ctx = {}
# 执行纯SQL语句的情景
q = Query("test", "test_result", Student,
"select * from student order by id")
q(ctx)
self._check_students_list(ctx["test_result"])
# 进行auto map的情况
q = Query("test", "test_result", "student",
"select * from student order by id")
q(ctx)
self._check_students_list(ctx["test_result"])
# 多个映射类的情况
q = Query("test", "test_result",
(Student.id, Student.name, Grade.name),
("select s.id, s.name, g.name from student s, grade g "
"where s.grade = g.id order by s.id"))
q(ctx)
self.assertTrue(ctx["test_result"][0], (1, "Sam", "Grade One"))
# auto map field 的情况
q = Query("test", "test_result",
(Student.id, "student.name", "grade.name"),
("select s.id, s.name, g.name from student s, grade g "
"where s.grade = g.id order by s.id"))
q(ctx)
self.assertTrue(ctx["test_result"][0], (1, "Sam", "Grade One"))
# 查询字符串表达式
q = Query("test",
"test_result",
Student,
"id > :id and name = :name",
params={
"id": 1,
"name": "Betty"
})
q(ctx)
students = ctx["test_result"]
self.assertEquals(len(students), 1)
s = students[0]
self.assertEquals((s.id, s.name, s.grade), (3, "Betty", 1))
# order by
q = Query("test",
"test_result",
Student,
"id > :id",
order_by="id desc",
params={"id": 1})
q(ctx)
self.assertEquals([std.id for std in ctx["test_result"]], [3, 2])
# group by and count
q = Query("test",
"test_result", ("student.grade", func.count('*')),
order_by="grade desc",
group_by="student.grade")
q(ctx)
grade_counts = ctx["test_result"]
self.assertEquals(len(grade_counts), 2)
self.assertEquals(grade_counts[0], (2, 1))
# 条件查询
q = Query("test", "test_result", Student,
and_(Student.grade == 1, Student.name == "Betty"))
q(ctx)
records = ctx["test_result"]
self.assertEquals(len(records), 1)
# 通过自定义函数来构建查询条件
def my_query(session, ctx, student, grade):
query = session.query(student.id,
grade.name).filter(student.grade == grade.id)
return query.all()
# 只返回查询对象
q_a = Query(
"test", "test_result",
("student", "grade"), lambda s, ctx, std, grade: s.query(
std.id, grade.name).filter(std.grade == grade.id))
# 通过all()返回查询结果
q_b = Query(
"test", "test_result",
("student", "grade"), lambda s, ctx, std, grade: s.query(
std.id, grade.name).filter(std.grade == grade.id).all())
for q in (q_a, q_b):
q(ctx)
records = ctx["test_result"]
self.assertEquals(len(records), 3)
self.assertEquals(records[0], (1, "Grade One"))
class StudentGrade(object):
def __init__(self, row):
student, grade = row
self.id = student.id
self.name = student.name
self.grade = grade.name
# row_handler
q = Query("test",
"test_result", (Student, Grade),
Student.grade == Grade.id,
row_handler=StudentGrade,
order_by=Student.id)
q(ctx)
records = ctx["test_result"]
self.assertEquals(len(records), 3)
self.assertEquals((records[0].id, records[0].name, records[0].grade),
(1, "Sam", "Grade One"))
# Table Wrapper
q = Query("test",
"test_table", (Student.id, Student.name, Student.grade),
result_wrapper=TableWrapper(
"student", (Title("id", u"编号"), Title(
"name", u"姓名"), Title("grade", u"年级"))))
q(ctx)
test_table = ctx["test_table"]
self.assertEquals(test_table.cell(0, 0), 1)
self.assertEquals(tuple(test_table.row(0)), (1, u"Sam", 1))
# Pure SQL Query
s = SQL("test", "sql_1", "select * from student order by id")
s(ctx)
records = ctx["sql_1"]
self.assertEquals(records[0], (1, "Sam", "SamChi", 1))
def my_row_handler(row):
return Student(id=row[0],
name=row[1],
fullname=row[2],
grade=row[3])
s = SQL("test",
"sql_2",
"select * from student where id > :id", {"id": 1},
row_handler=my_row_handler)
s(ctx)
records = ctx["sql_2"]
self.assertEquals((records[0].id, records[0].name), (2, "Jack"))
# Table Wrapper
s = SQL("test",
"sql_3",
"select * from student order by id",
result_wrapper=TableWrapper(
"students",
(Title("id", u"编号"), Title("name", u"名字"),
Title("fullname", u"全名"), Title("grade", u"年级"))))
s(ctx)
table = ctx["sql_3"]
row = table[0]
self.assertEquals((row.id, row.name), (1, "Sam"))
# 测试非查询语句
s = SQL("test", None,
("insert into student (id, name, fullname, grade) "
"values (:id, :name, :fullname, :grade)"), {
"id": 4,
"name": "Jane",
"fullname": "JaneGreen",
"grade": 3
})
s(ctx)
SQL("test", "sql_4", "select * from student where id = 4")(ctx)
self.assertEquals(ctx["sql_4"][0], (4, "Jane", "JaneGreen", 3))
# Test Plugin
plugin_manager.sys_prepare(self.config)
orm_query = plugin_manager.plugin("orm_query")
orm_query.execute(
ctx, Query("test", "result_1", (Student.id, Student.name),
"id = 1"),
Query("test", "result_2", (Student.id, Student.name), "id = 2"))
result_1, result_2 = ctx["result_1"], ctx["result_2"]
self.assertEquals([(1, "Sam")], result_1)
self.assertEquals([(2, "Jack")], result_2)
def test_read_one_line(self):
# only read line
ctx = self.workflow_context()
records = TextR(filepath="test.log", record_matcher="line")(ctx)
self.assertEquals(len(records), len(log_content.splitlines()))
self.assertTrue(records[-1].startswith("OperationalError:"))
# with max lines
records = TextR(filepath="test.log", record_matcher="line",
max_line=2)(ctx)
self.assertEquals(len(records), 2)
# with record_filter and record_handler
records = TextR(
filepath="test.log",
record_matcher="line",
record_filter=lambda line: bool(re.search("^2016", line)),
record_handler=lambda line: tuple(line.split(" - ")))(ctx)
self.assertEquals(records[-1][1:3], (u"girlfriend", u"ERROR"))
# with result wrapper
table = TextR(
filepath="test.log",
record_matcher="line",
record_filter=lambda line: bool(re.search("^2016", line)),
record_handler=lambda line: tuple(line.split(" - ")),
result_wrapper=TableWrapper("log_table",
("time", u"时间", "logger", u"日志",
"level", u"级别", "msg", u"信息")))(ctx)
print
print table
self.assertEquals(table[0][("logger", "level")],
(u"girlfriend", u"INFO"))
# with pointer
records = TextR(filepath="test.log",
record_matcher="line",
pointer="test_pointer")(ctx)
with open("test.log", "a") as f:
for i in xrange(0, 10):
f.write(str(i) + "\n")
records = TextR(filepath="test.log",
record_matcher="line",
record_handler=int,
pointer="test_pointer")(ctx)
self.assertEquals(records, range(0, 10))
# with file change
with open("test.log", "a") as f:
for i in xrange(10, 20):
f.write(str(i) + "\n")
os.rename("test.log", "test.log.bak")
with open("test.log", "w") as f:
for i in xrange(20, 30):
f.write(str(i) + "\n")
records = TextR(filepath="test.log",
record_matcher="line",
record_handler=int,
pointer="test_pointer",
change_file_logic=lambda ctx: "test.log.bak")(ctx)
self.assertEquals(records, range(10, 30))