def convert_csv(self, csv, table=DEFAULT_TABLE_NAME, headers=None, types=None): '''Converts a CSV to a SQLite database and returns a database cursor into the resulting file''' dbpath = TEMP_DB_PATH if headers is not None: try: headerObj = StringIO(headers) except TypeError: headerObj = BytesIO(headers) else: headerObj = None if types is not None: try: typeObj = StringIO(types) except TypeError: typeObj = BytesIO(types) else: typeObj = None try: infile = StringIO(csv) # Python 3 except: infile = BytesIO(csv) # Python 2 convert(infile, dbpath, table, headerObj, None, typeObj) conn = sqlite3.connect(dbpath) c = conn.cursor() return c
def convert_csv(self, csv, table = DEFAULT_TABLE_NAME, headers = None, types = None): '''Converts a CSV to a SQLite database and returns a database cursor into the resulting file''' dbpath = TEMP_DB_PATH if headers is not None: try: headerObj = StringIO(headers) except TypeError: headerObj = BytesIO(headers) else: headerObj = None if types is not None: try: typeObj = StringIO(types) except TypeError: typeObj = BytesIO(types) else: typeObj = None try: infile = StringIO(csv) # Python 3 except: infile = BytesIO(csv) # Python 2 convert(infile, dbpath, table, headerObj, None, typeObj) conn = sqlite3.connect(dbpath) c = conn.cursor() return c
def convert_csv(self, csv, table = DEFAULT_TABLE_NAME): '''Converts a CSV file to a SQLite database and returns a database cursor into the resulting file''' dbpath = TEMP_DB_PATH convert(StringIO.StringIO(csv), dbpath, table) conn = sqlite3.connect(dbpath) c = conn.cursor() return c
def convert_file(self, path, table = DEFAULT_TABLE_NAME, compression = None): '''Converts a CSV file to a SQLite database and returns a database cursor into the resulting file''' dbpath = TEMP_DB_PATH convert(path, dbpath, table, compression=compression) conn = sqlite3.connect(dbpath) c = conn.cursor() return c
def convert_csv(self, csv, table = DEFAULT_TABLE_NAME, headers = None): '''Converts a CSV to a SQLite database and returns a database cursor into the resulting file''' dbpath = TEMP_DB_PATH headerObj = StringIO.StringIO(headers) if headers is not None else None convert(StringIO.StringIO(csv), dbpath, table, headerObj) conn = sqlite3.connect(dbpath) c = conn.cursor() return c
def test(): '''Simple test case''' import StringIO import os fileobj = StringIO.StringIO('''heading_1,heading_2,heading_3 abc,1,1.0 xyz,2,2.0 efg,3,3.0''') dbpath = '/tmp/csv2sqlite-test-data.db' if os.path.exists(dbpath): os.remove(dbpath) table = 'data' convert(fileobj, dbpath, table) conn = sqlite3.connect(dbpath) c = conn.cursor() c.execute('select count(heading_3) from %s' % table) row = c.next() assert row[0] == 3, row
def test_semicolon(): '''Simple test case (semicolon)''' import StringIO import os fileobj = StringIO.StringIO('''heading_1;heading_2;heading_3 abc;1;1,0 xyz;2;2,0 efg;3;3,0''') dbpath = '/tmp/csv2sqlite-test-data.db' if os.path.exists(dbpath): os.remove(dbpath) table = 'data' convert(fileobj, dbpath, table) conn = sqlite3.connect(dbpath) c = conn.cursor() c.execute('select count(heading_3) from %s' % table) row = c.next() assert row[0] == 3, row
def test(): '''Simple test case''' import StringIO import os fileobj = StringIO.StringIO( '''heading_1,heading_2,heading_3 abc,1,1.0 xyz,2,2.0 efg,3,3.0''' ) dbpath = '/tmp/csv2sqlite-test-data.db' if os.path.exists(dbpath): os.remove(dbpath) table = 'data' convert(fileobj, dbpath, table) conn = sqlite3.connect(dbpath) c = conn.cursor() c.execute('select count(heading_3) from %s' % table); row = c.next() assert row[0] == 3, row
def test_semicolon(): '''Simple test case (semicolon)''' import StringIO import os fileobj = StringIO.StringIO( '''heading_1;heading_2;heading_3 abc;1;1,0 xyz;2;2,0 efg;3;3,0''' ) dbpath = '/tmp/csv2sqlite-test-data.db' if os.path.exists(dbpath): os.remove(dbpath) table = 'data' convert(fileobj, dbpath, table) conn = sqlite3.connect(dbpath) c = conn.cursor() c.execute('select count(heading_3) from %s' % table); row = c.next() assert row[0] == 3, row
from csv2sqlite import convert import sqlite3 import os import sys if __name__ == '__main__': dbpath = 'profile.db' if os.path.exists(dbpath): os.remove(dbpath) filename = sys.argv[1] table_name = filename if 1: #for table_name in ( 'bio', 'universities', 'education', 'groups' ): table = table_name fileobj = filename convert(fileobj, dbpath, table) conn = sqlite3.connect(dbpath) c = conn.cursor() c.execute('select count(*) from %s' % table); row = c.next() #assert row[0] == 3, row
from csv2sqlite import convert import sqlite3 import os if __name__ == '__main__': dbpath = 'profile.db' if os.path.exists(dbpath): os.remove(dbpath) for table_name in ( 'bio', 'universities', 'education', 'groups' ): table = table_name fileobj = 'bio_assignment - %s.csv' % table_name convert(fileobj, dbpath, table) conn = sqlite3.connect(dbpath) c = conn.cursor() c.execute('select count(*) from %s' % table); row = c.next() #assert row[0] == 3, row
global x soup = BeautifulSoup(ss, 'html.parser') for data in soup.findAll("tr", bgcolor="#ccffff"): row = [s.text for s in data.findAll("td")] out.writerow(row) x += 1 while True: if ke > 1: w = "https://www.nomor.net/_kodepos.php?_i=desa-kodepos&daerah=&jobs=&perhal=%s&urut=8&asc=0001000&sby=000000&no1=%s&no2=%s&kk=%s" % ( perpage, (perpage * (ke - 2)) + 1, ((perpage * (ke - 1)) + 1) - 1, ke) else: w = "https://www.nomor.net/_kodepos.php?_i=desa-kodepos&daerah=&jobs=&perhal=%s&sby=000000&asc=0001000&urut=8" % perpage c = requests.get(w, headers=headers).text print(c) if c.find("#ccffff") < 1 or ke > limitpage: break parse(c) ke += 1 f.close() endTime = datetime.now() - startTime print("Convert into sqlite...") csv2sqlite.convert(os.path.join(mydir, fname), os.path.join(mydir, os.path.splitext(fname)[0] + ".db"), "kodepos") print("All Done, %s data downloaded. time %s" % (x, endTime))