def easy_way():
metadata = MetaData()
engine = Sqlite3Engine(":memory:")
datatype = DataType()
employee = Table("employee", metadata,
Column("employee_id", datatype.text, primary_key=True),
Column("age", datatype.integer),
Column("height", datatype.real),
Column("enroll_date", datatype.date),
Column("create_datetime", datatype.datetime),
)
metadata.create_all(engine)
ins = employee.insert()
timer.start()
# === real work ===
sheet = Sheet({"employee_id": "TEXT", "age": "INTEGER", "height": "REAL",
"enroll_date": "DATE", "create_datetime": "DATETIME"})
csvfile = CSVFile(r"testdata\bigcsvfile.txt",
sep=",",
header=True,
dtype={"CREATE_DATE": "DATE", "CREATE_DATETIME": "DATETIME"})
sheet.match(csvfile.sample)
for row in csvfile.generate_rows():
try:
engine.insert_row(ins, Row.from_dict(sheet.convert(row)))
except:
pass
timer.timeup()
engine.prt_all(employee)
def easy_way():
metadata = MetaData()
engine = Sqlite3Engine(":memory:")
datatype = DataType()
employee = Table(
"employee",
metadata,
Column("employee_id", datatype.text, primary_key=True),
Column("age", datatype.integer),
Column("height", datatype.real),
Column("enroll_date", datatype.date),
Column("create_datetime", datatype.datetime),
)
metadata.create_all(engine)
ins = employee.insert()
timer.start()
# === real work ===
sheet = Sheet({
"employee_id": "TEXT",
"age": "INTEGER",
"height": "REAL",
"enroll_date": "DATE",
"create_datetime": "DATETIME"
})
csvfile = CSVFile(r"testdata\bigcsvfile.txt",
sep=",",
header=True,
dtype={
"CREATE_DATE": "DATE",
"CREATE_DATETIME": "DATETIME"
})
sheet.match(csvfile.sample)
for row in csvfile.generate_rows():
try:
engine.insert_row(ins, Row.from_dict(sheet.convert(row)))
except:
pass
timer.timeup()
engine.prt_all(employee)
df = [["eid001", 100, 1.23, date.today(), datetime.now()]]
df = pd.DataFrame(df, columns=["TEXT", "INTEGER", "REAL", "DATE", "DATETIME"])
df.to_csv(r"testdata\with_header.txt", index=False)
df.to_csv(r"testdata\without_header.txt", header=False, index=False)
# create_testdata()
# 定义你目标数据格式
sheet = Sheet({"_id": "TEXT", "age": "INTEGER", "height": "REAL",
"create_date": "DATE", "datetime": "DATETIME"})
csvfile_with_header = CSVFile(r"testdata\with_header.txt",
sep=",",
header=True,
usecols=[0,1,2,3,4],
dtype={"DATE": "DATE", "DATETIME": "DATETIME"})
sheet.match(csvfile_with_header.sample)
for row in csvfile_with_header.generate_rows():
print( sheet.convert(row) )
csvfile_without_header = CSVFile(r"testdata\without_header.txt",
sep=",",
header=False,
usecols=[0,1,2,3,4],
dtype={3: "DATE", 4: "DATETIME"})
sheet.match(csvfile_without_header.sample)
for row in csvfile_without_header.generate_rows():
print( sheet.convert(row) )
print("Complete")
"_id": "TEXT",
"age": "INTEGER",
"height": "REAL",
"create_date": "DATE",
"datetime": "DATETIME"
})
csvfile_with_header = CSVFile(r"testdata\with_header.txt",
sep=",",
header=True,
usecols=[0, 1, 2, 3, 4],
dtype={
"DATE": "DATE",
"DATETIME": "DATETIME"
})
sheet.match(csvfile_with_header.sample)
for row in csvfile_with_header.generate_rows():
print(sheet.convert(row))
csvfile_without_header = CSVFile(r"testdata\without_header.txt",
sep=",",
header=False,
usecols=[0, 1, 2, 3, 4],
dtype={
3: "DATE",
4: "DATETIME"
})
sheet.match(csvfile_without_header.sample)
for row in csvfile_without_header.generate_rows():
print(sheet.convert(row))
