def easy_way(): metadata = MetaData() engine = Sqlite3Engine(":memory:") datatype = DataType() employee = Table("employee", metadata, Column("employee_id", datatype.text, primary_key=True), Column("age", datatype.integer), Column("height", datatype.real), Column("enroll_date", datatype.date), Column("create_datetime", datatype.datetime), ) metadata.create_all(engine) ins = employee.insert() timer.start() # === real work === sheet = Sheet({"employee_id": "TEXT", "age": "INTEGER", "height": "REAL", "enroll_date": "DATE", "create_datetime": "DATETIME"}) csvfile = CSVFile(r"testdata\bigcsvfile.txt", sep=",", header=True, dtype={"CREATE_DATE": "DATE", "CREATE_DATETIME": "DATETIME"}) sheet.match(csvfile.sample) for row in csvfile.generate_rows(): try: engine.insert_row(ins, Row.from_dict(sheet.convert(row))) except: pass timer.timeup() engine.prt_all(employee)
def easy_way(): metadata = MetaData() engine = Sqlite3Engine(":memory:") datatype = DataType() employee = Table( "employee", metadata, Column("employee_id", datatype.text, primary_key=True), Column("age", datatype.integer), Column("height", datatype.real), Column("enroll_date", datatype.date), Column("create_datetime", datatype.datetime), ) metadata.create_all(engine) ins = employee.insert() timer.start() # === real work === sheet = Sheet({ "employee_id": "TEXT", "age": "INTEGER", "height": "REAL", "enroll_date": "DATE", "create_datetime": "DATETIME" }) csvfile = CSVFile(r"testdata\bigcsvfile.txt", sep=",", header=True, dtype={ "CREATE_DATE": "DATE", "CREATE_DATETIME": "DATETIME" }) sheet.match(csvfile.sample) for row in csvfile.generate_rows(): try: engine.insert_row(ins, Row.from_dict(sheet.convert(row))) except: pass timer.timeup() engine.prt_all(employee)
df = [["eid001", 100, 1.23, date.today(), datetime.now()]] df = pd.DataFrame(df, columns=["TEXT", "INTEGER", "REAL", "DATE", "DATETIME"]) df.to_csv(r"testdata\with_header.txt", index=False) df.to_csv(r"testdata\without_header.txt", header=False, index=False) # create_testdata() # 定义你目标数据格式 sheet = Sheet({"_id": "TEXT", "age": "INTEGER", "height": "REAL", "create_date": "DATE", "datetime": "DATETIME"}) csvfile_with_header = CSVFile(r"testdata\with_header.txt", sep=",", header=True, usecols=[0,1,2,3,4], dtype={"DATE": "DATE", "DATETIME": "DATETIME"}) sheet.match(csvfile_with_header.sample) for row in csvfile_with_header.generate_rows(): print( sheet.convert(row) ) csvfile_without_header = CSVFile(r"testdata\without_header.txt", sep=",", header=False, usecols=[0,1,2,3,4], dtype={3: "DATE", 4: "DATETIME"}) sheet.match(csvfile_without_header.sample) for row in csvfile_without_header.generate_rows(): print( sheet.convert(row) ) print("Complete")
"_id": "TEXT", "age": "INTEGER", "height": "REAL", "create_date": "DATE", "datetime": "DATETIME" }) csvfile_with_header = CSVFile(r"testdata\with_header.txt", sep=",", header=True, usecols=[0, 1, 2, 3, 4], dtype={ "DATE": "DATE", "DATETIME": "DATETIME" }) sheet.match(csvfile_with_header.sample) for row in csvfile_with_header.generate_rows(): print(sheet.convert(row)) csvfile_without_header = CSVFile(r"testdata\without_header.txt", sep=",", header=False, usecols=[0, 1, 2, 3, 4], dtype={ 3: "DATE", 4: "DATETIME" }) sheet.match(csvfile_without_header.sample) for row in csvfile_without_header.generate_rows(): print(sheet.convert(row))