def do_insert():
if values:
connection.execute(insert, *values)
del values[:]
percent = float(steps_so_far) / float(total_lines)
if 0 < percent < 1.0:
pb_set_fraction(percent)
def do_insert():
if values:
connection.execute(insert, *values)
del values[:]
percent = float(steps_so_far)/float(total_lines)
if 0 < percent < 1.0:
pb_set_fraction(percent)
def __export_task(self, path):
# if not os.path.exists(path):
# raise ValueError("CSVExporter: path does not exist.\n" + path)
filename_template = os.path.join(path, "%s.txt")
# timeout = tasklet.WaitForTimeout(12)
steps_so_far = 0
ntables = 0
for table in db.metadata.sorted_tables:
ntables += 1
filename = filename_template % table.name
if os.path.exists(filename):
msg = _(
"Export file <b>%(filename)s</b> for "
"<b>%(table)s</b> table already exists.\n\n<i>Would "
"you like to continue?</i>"
) % {"filename": filename, "table": table.name}
if utils.yes_no_dialog(msg):
return
def replace(s):
if isinstance(s, (str, unicode)):
s.replace("\n", "\\n")
return s
def write_csv(filename, rows):
f = open(filename, "wb")
writer = UnicodeWriter(f, quotechar=QUOTE_CHAR, quoting=QUOTE_STYLE)
writer.writerows(rows)
f.close()
update_every = 30
for table in db.metadata.sorted_tables:
filename = filename_template % table.name
steps_so_far += 1
fraction = float(steps_so_far) / float(ntables)
pb_set_fraction(fraction)
msg = _("exporting %(table)s table to %(filename)s") % {"table": table.name, "filename": filename}
bauble.task.set_message(msg)
logger.info("exporting %s" % table.name)
# get the data
results = table.select().execute().fetchall()
# create empty files with only the column names
if len(results) == 0:
write_csv(filename, [table.c.keys()])
yield
continue
rows = []
rows.append(table.c.keys()) # append col names
ctr = 0
for row in results:
values = map(replace, row.values())
rows.append(values)
if ctr == update_every:
yield
ctr = 0
ctr += 1
write_csv(filename, rows)
def __export_task(self, path):
filename_template = os.path.join(path, "%s.txt")
steps_so_far = 0
ntables = 0
for table in db.metadata.sorted_tables:
ntables += 1
filename = filename_template % table.name
if os.path.exists(filename):
msg = _('Export file <b>%(filename)s</b> for '
'<b>%(table)s</b> table already exists.\n\n<i>Would '
'you like to continue?</i>')\
% {'filename': filename, 'table': table.name}
if utils.yes_no_dialog(msg):
return
def replace(s):
if isinstance(s, (str, unicode)):
s.replace('\n', '\\n')
return s
def write_csv(filename, rows):
f = open(filename, 'wb')
writer = UnicodeWriter(f,
quotechar=QUOTE_CHAR,
quoting=QUOTE_STYLE)
writer.writerows(rows)
f.close()
update_every = 30
for table in db.metadata.sorted_tables:
filename = filename_template % table.name
steps_so_far += 1
fraction = float(steps_so_far) / float(ntables)
pb_set_fraction(fraction)
msg = _('exporting %(table)s table to %(filename)s')\
% {'table': table.name, 'filename': filename}
bauble.task.set_message(msg)
logger.info("exporting %s" % table.name)
# get the data
results = table.select().execute().fetchall()
# create empty files with only the column names
if len(results) == 0:
write_csv(filename, [table.c.keys()])
yield
continue
rows = []
rows.append(table.c.keys()) # append col names
ctr = 0
for row in results:
values = map(replace, row.values())
rows.append(values)
if ctr == update_every:
yield
ctr = 0
ctr += 1
write_csv(filename, rows)
def __export_task(self, path):
filename_template = os.path.join(path, "%s.txt")
steps_so_far = 0
ntables = 0
for table in db.metadata.sorted_tables:
ntables += 1
filename = filename_template % table.name
if os.path.exists(filename):
msg = _('Export file <b>%(filename)s</b> for '
'<b>%(table)s</b> table already exists.\n\n<i>Would '
'you like to continue?</i>')\
% {'filename': filename, 'table': table.name}
if utils.yes_no_dialog(msg):
return
def replace(s):
if isinstance(s, (str, unicode)):
s.replace('\n', '\\n')
return s
def write_csv(filename, rows):
f = open(filename, 'wb')
writer = UnicodeWriter(f, quotechar=QUOTE_CHAR,
quoting=QUOTE_STYLE)
writer.writerows(rows)
f.close()
update_every = 30
for table in db.metadata.sorted_tables:
filename = filename_template % table.name
steps_so_far += 1
fraction = float(steps_so_far)/float(ntables)
pb_set_fraction(fraction)
msg = _('exporting %(table)s table to %(filename)s')\
% {'table': table.name, 'filename': filename}
bauble.task.set_message(msg)
logger.info("exporting %s" % table.name)
# get the data
results = table.select().execute().fetchall()
# create empty files with only the column names
if len(results) == 0:
write_csv(filename, [table.c.keys()])
yield
continue
rows = []
rows.append(table.c.keys()) # append col names
ctr = 0
for row in results:
values = map(replace, row.values())
rows.append(values)
if ctr == update_every:
yield
ctr = 0
ctr += 1
write_csv(filename, rows)
def run(self, objects):
## generator function. will be run as a task.
session = db.Session()
n = len(objects)
for i, obj in enumerate(objects):
try:
db.construct_from_dict(session, obj, self.create, self.update)
except Exception as e:
logger.warning("could not import %s (%s: %s)" %
(obj, type(e).__name__, e.args))
pb_set_fraction(float(i) / n)
yield
session.commit()
