def loaddata(datadir):
create_tables()
model_classes = [Party, County, ElectionType, Election, OfficeType, Office,
Candidate, Vote, VoteTotal]
reader_kwargs = {
'delimiter': '\t',
}
for model_cls in model_classes:
infilename = os.path.join(datadir, raw_filename(model_cls))
load_csv(model_cls, os.path.join(datadir, infilename),
has_header=True, **reader_kwargs)
def loaddata(datadir):
create_tables()
model_classes = [Party, County, ElectionType, Election, OfficeType, Office,
Candidate, Vote, VoteTotal]
reader_kwargs = {
'delimiter': '\t',
}
for model_cls in model_classes:
infilename = os.path.join(datadir, raw_filename(model_cls))
infilepath = os.path.join(datadir, infilename)
logging.info("Loading input file {}".format(infilepath))
load_csv(model_cls, infilepath, has_header=True, **reader_kwargs)
Election.parse_electiondates()
def setup_urls():
file_with_url = os.path.join("tmp",
"file_on_{}".format(UrlBuilder.SERVER_NAME))
cmd = [
"bash", "-c",
"rsync -a --list-only {} > {}".format(UrlBuilder.RSYNC, file_with_url)
]
exec_cmd(cmd)
in_place_opt = ["-i", ".bak"] if platform.system() == "Darwin" else ["-i"]
cmd = ["sed"] + in_place_opt + [r"s#.* \(.*\)$#\\1#", file_with_url]
exec_cmd(cmd)
field_names = ['url']
load_csv(Url, file_with_url, field_names=field_names)
def loaddata(datadir):
create_tables()
model_classes = [
Party, County, ElectionType, Election, OfficeType, Office, Candidate,
Vote, VoteTotal
]
reader_kwargs = {
'delimiter': '\t',
}
for model_cls in model_classes:
infilename = os.path.join(datadir, raw_filename(model_cls))
infilepath = os.path.join(datadir, infilename)
logging.info("Loading input file {}".format(infilepath))
load_csv(model_cls, infilepath, has_header=True, **reader_kwargs)
Election.parse_electiondates()
def setup_urls():
file_with_url = os.path.join("tmp",
"file_on_{}".format(UrlBuilder.SERVER_NAME))
cmd = [
"bash",
"-c",
"rsync -a --list-only {} > {}".format(UrlBuilder.RSYNC, file_with_url),
]
exec_cmd(cmd)
# make a copy of rsync's result
shutil.copyfile(file_with_url, file_with_url + ".bak")
# strip rsync file to only contain relative path
with open(file_with_url + ".bak", "r") as src, open(file_with_url,
"w") as dest:
for line in src.readlines():
if len(line) >= 47:
dest.write(line[46:])
field_names = ["url"]
load_csv(Url, file_with_url, field_names=field_names)
def import_(self, directory_name):
import_dir = join(self.export_base_dir, directory_name)
if not (exists(join(import_dir, 'sessions.csv')) and
exists(join(import_dir, 'members.csv'))):
toast('Invalid backup')
return
try:
with database_proxy.atomic():
load_csv(Session, join(import_dir, 'sessions.csv'), [
Session.id,
Session.name,
Session.date
])
load_csv(Member, join(import_dir, 'members.csv'))
for session in glob(join(import_dir, 'session_*.csv')):
load_csv(SessionAttendance, join(import_dir, session))
except IntegrityError:
toast('Data already imported, canceling')
else:
toast('Data Successfully imported')
def create_demo_db():
if not ENVARS.is_demo_web_server():
raise Exception('wrong database! call this again with FGC_DEMO_DB=1')
DATABASE.create_tables([Player, Match, Rank])
load_csv(Rank, 'temp/rank.csv')
load_csv(Player, 'temp/player.csv')
from models import FuelEconomy from playhouse.csv_loader import load_csv from sys import argv load_csv(FuelEconomy, argv[1])
# DESCRIPTION
# This small script will turn the Ukrainian small business taxpayers CSV database
# into a much more accessible Sqlite3 format. Feel free to change whatever.
# N.B. I know there are other ways to do this. I just find this more convenient.
# Instructions:
# 1. Download the database file (should be available at http://data.gov.ua/passport/5fc89a6f-55b8-4ec6-95d6-38a0fdc31be1)
# what you need is "Відомості щодо юридичних осіб".
# 2. Extract the zip file. You should have a file called UO.csv or something like that.
# 3. Now we need to fix the encoding mess.
# - On a Linux PC (on Windows use iconv.exe) do this:
# iconv -f CP1251 -t UTF-8 UO.csv > uo-utf.csv
# This will save a utf-8 encoded copy of the file.
# Just in case, more info is available here: http://stackoverflow.com/questions/15422753/iconv-convert-from-cp1252-to-utf-8
from peewee import *
from playhouse.csv_loader import load_csv
db = SqliteDatabase('payers.db')
fields = [CharField(null=True), CharField(null=True), IntegerField(null=True), CharField(null=True), CharField(null=True), CharField(null=True), CharField(null=True)]
field_names = ['name', 'short_name', 'taxpayer_code', 'address', 'director_name', 'major_activity', 'status']
print 'This script takes a while (like 10-15 min maybe?), so please, be patient.'
print "You can watch the payers.db file - if it's size changes, then all is going well and you just need to wait."
data = load_csv(db, 'uoutf.csv', fields=fields, field_names=field_names, has_header=True, delimiter=';')
print ''
print 'Done!'
def upload_csv(model, path):
"""Uploads a csv file into the DB"""
load_csv(model, path)
def seed(fname):
load_csv(Word, fname)