def convert_csv(self,
csv,
table=DEFAULT_TABLE_NAME,
headers=None,
types=None):
'''Converts a CSV to a SQLite database and returns a database cursor
into the resulting file'''
dbpath = TEMP_DB_PATH
if headers is not None:
try:
headerObj = StringIO(headers)
except TypeError:
headerObj = BytesIO(headers)
else:
headerObj = None
if types is not None:
try:
typeObj = StringIO(types)
except TypeError:
typeObj = BytesIO(types)
else:
typeObj = None
try:
infile = StringIO(csv) # Python 3
except:
infile = BytesIO(csv) # Python 2
convert(infile, dbpath, table, headerObj, None, typeObj)
conn = sqlite3.connect(dbpath)
c = conn.cursor()
return c
def convert_csv(self, csv, table = DEFAULT_TABLE_NAME, headers = None, types = None):
'''Converts a CSV to a SQLite database and returns a database cursor
into the resulting file'''
dbpath = TEMP_DB_PATH
if headers is not None:
try:
headerObj = StringIO(headers)
except TypeError:
headerObj = BytesIO(headers)
else:
headerObj = None
if types is not None:
try:
typeObj = StringIO(types)
except TypeError:
typeObj = BytesIO(types)
else:
typeObj = None
try:
infile = StringIO(csv) # Python 3
except:
infile = BytesIO(csv) # Python 2
convert(infile, dbpath, table, headerObj, None, typeObj)
conn = sqlite3.connect(dbpath)
c = conn.cursor()
return c
def convert_csv(self, csv, table = DEFAULT_TABLE_NAME):
'''Converts a CSV file to a SQLite database and returns a database cursor
into the resulting file'''
dbpath = TEMP_DB_PATH
convert(StringIO.StringIO(csv), dbpath, table)
conn = sqlite3.connect(dbpath)
c = conn.cursor()
return c
def convert_file(self, path, table = DEFAULT_TABLE_NAME, compression = None):
'''Converts a CSV file to a SQLite database and returns a database cursor
into the resulting file'''
dbpath = TEMP_DB_PATH
convert(path, dbpath, table, compression=compression)
conn = sqlite3.connect(dbpath)
c = conn.cursor()
return c
def convert_csv(self, csv, table = DEFAULT_TABLE_NAME, headers = None):
'''Converts a CSV to a SQLite database and returns a database cursor
into the resulting file'''
dbpath = TEMP_DB_PATH
headerObj = StringIO.StringIO(headers) if headers is not None else None
convert(StringIO.StringIO(csv), dbpath, table, headerObj)
conn = sqlite3.connect(dbpath)
c = conn.cursor()
return c
def test():
'''Simple test case'''
import StringIO
import os
fileobj = StringIO.StringIO('''heading_1,heading_2,heading_3
abc,1,1.0
xyz,2,2.0
efg,3,3.0''')
dbpath = '/tmp/csv2sqlite-test-data.db'
if os.path.exists(dbpath):
os.remove(dbpath)
table = 'data'
convert(fileobj, dbpath, table)
conn = sqlite3.connect(dbpath)
c = conn.cursor()
c.execute('select count(heading_3) from %s' % table)
row = c.next()
assert row[0] == 3, row
def test_semicolon():
'''Simple test case (semicolon)'''
import StringIO
import os
fileobj = StringIO.StringIO('''heading_1;heading_2;heading_3
abc;1;1,0
xyz;2;2,0
efg;3;3,0''')
dbpath = '/tmp/csv2sqlite-test-data.db'
if os.path.exists(dbpath):
os.remove(dbpath)
table = 'data'
convert(fileobj, dbpath, table)
conn = sqlite3.connect(dbpath)
c = conn.cursor()
c.execute('select count(heading_3) from %s' % table)
row = c.next()
assert row[0] == 3, row
def test():
'''Simple test case'''
import StringIO
import os
fileobj = StringIO.StringIO(
'''heading_1,heading_2,heading_3
abc,1,1.0
xyz,2,2.0
efg,3,3.0'''
)
dbpath = '/tmp/csv2sqlite-test-data.db'
if os.path.exists(dbpath):
os.remove(dbpath)
table = 'data'
convert(fileobj, dbpath, table)
conn = sqlite3.connect(dbpath)
c = conn.cursor()
c.execute('select count(heading_3) from %s' % table);
row = c.next()
assert row[0] == 3, row
def test_semicolon():
'''Simple test case (semicolon)'''
import StringIO
import os
fileobj = StringIO.StringIO(
'''heading_1;heading_2;heading_3
abc;1;1,0
xyz;2;2,0
efg;3;3,0'''
)
dbpath = '/tmp/csv2sqlite-test-data.db'
if os.path.exists(dbpath):
os.remove(dbpath)
table = 'data'
convert(fileobj, dbpath, table)
conn = sqlite3.connect(dbpath)
c = conn.cursor()
c.execute('select count(heading_3) from %s' % table);
row = c.next()
assert row[0] == 3, row
from csv2sqlite import convert
import sqlite3
import os
import sys
if __name__ == '__main__':
dbpath = 'profile.db'
if os.path.exists(dbpath):
os.remove(dbpath)
filename = sys.argv[1]
table_name = filename
if 1:
#for table_name in ( 'bio', 'universities', 'education', 'groups' ):
table = table_name
fileobj = filename
convert(fileobj, dbpath, table)
conn = sqlite3.connect(dbpath)
c = conn.cursor()
c.execute('select count(*) from %s' % table);
row = c.next()
#assert row[0] == 3, row
from csv2sqlite import convert
import sqlite3
import os
if __name__ == '__main__':
dbpath = 'profile.db'
if os.path.exists(dbpath):
os.remove(dbpath)
for table_name in ( 'bio', 'universities', 'education', 'groups' ):
table = table_name
fileobj = 'bio_assignment - %s.csv' % table_name
convert(fileobj, dbpath, table)
conn = sqlite3.connect(dbpath)
c = conn.cursor()
c.execute('select count(*) from %s' % table);
row = c.next()
#assert row[0] == 3, row
global x
soup = BeautifulSoup(ss, 'html.parser')
for data in soup.findAll("tr", bgcolor="#ccffff"):
row = [s.text for s in data.findAll("td")]
out.writerow(row)
x += 1
while True:
if ke > 1:
w = "https://www.nomor.net/_kodepos.php?_i=desa-kodepos&daerah=&jobs=&perhal=%s&urut=8&asc=0001000&sby=000000&no1=%s&no2=%s&kk=%s" % (
perpage, (perpage * (ke - 2)) + 1,
((perpage * (ke - 1)) + 1) - 1, ke)
else:
w = "https://www.nomor.net/_kodepos.php?_i=desa-kodepos&daerah=&jobs=&perhal=%s&sby=000000&asc=0001000&urut=8" % perpage
c = requests.get(w, headers=headers).text
print(c)
if c.find("#ccffff") < 1 or ke > limitpage:
break
parse(c)
ke += 1
f.close()
endTime = datetime.now() - startTime
print("Convert into sqlite...")
csv2sqlite.convert(os.path.join(mydir, fname),
os.path.join(mydir,
os.path.splitext(fname)[0] + ".db"), "kodepos")
print("All Done, %s data downloaded. time %s" % (x, endTime))
