def execute_statement(self, conn: LoggedConnection, fname: str, line: int,
statement: SQL, run_params: Params,
ignore_error: bool) -> bool:
'''Log and execute one statement.
'''
sqlerror = Script.sqlerror(statement)
if sqlerror is not None:
return sqlerror
params = params_used(run_params, statement)
self.set_status_message('%s:%s:\n%s\n%s' %
(fname, line, statement, params))
conn.execute(statement, params)
return ignore_error
def execute_statement(self, conn: LoggedConnection, fname: str, line: int,
statement: SQL, run_params: Params,
ignore_error: bool) -> bool:
'''Log and execute one statement.
'''
sqlerror = Script.sqlerror(statement)
if sqlerror is not None:
return sqlerror
params = params_used(run_params, statement)
# This seems to trigger:
# 14:47:06 30165 ERROR: Uncaught exception in luigi
# luigi.rpc.RPCError: Received null response from remote scheduler
# self.set_status_message(
# '%s:%s:\n%s\n%s' % (fname, line, statement, params))
conn.execute(statement, params)
return ignore_error
def complete(self) -> bool:
'''Each script's last query tells whether it is complete.
It should be a scalar query that returns non-zero for done
and either zero or an error for not done.
'''
last_query = self.last_query()
params = params_used(self.complete_params(), last_query)
with self.connection(event=self.task_family + ' complete query: ' + self.script.name) as conn:
try:
result = conn.scalar(sql_text(last_query), params)
return bool(result)
except DatabaseError as exc:
conn.log.warning('%(event)s: %(exc)s',
dict(event='complete query error', exc=exc))
return False
def bulk_insert(self, conn: LoggedConnection, fname: str, line: int,
statement: SQL, run_params: Params, bulk_rows: int) -> int:
with conn.log.step(
'%(filename)s:%(lineno)s: %(event)s',
dict(event='bulk_insert', filename=fname,
lineno=line)) as step:
plan = '\n'.join(explain_plan(conn, statement))
conn.log.info(
'%(filename)s:%(lineno)s: %(event)s:\n%(plan)s',
dict(filename=fname, lineno=line, event='plan', plan=plan))
params = params_used(run_params, statement)
self.set_status_message('%s:%s:\n%s\n%s' %
(fname, line, statement, params))
last_result = conn.execute(statement, params)
step.msg_parts.append(' %(rowcount)d rows')
step.argobj.update(dict(rowcount=last_result.rowcount))
bulk_rows += last_result.rowcount
return bulk_rows
def bulk_insert(self, conn: LoggedConnection, fname: str, line: int,
statement: SQL, run_params: Params,
bulk_rows: int) -> int:
with conn.log.step(
'%(filename)s:%(lineno)s: %(event)s',
dict(event='bulk_insert',
filename=fname, lineno=line)):
plan = '\n'.join(explain_plan(conn, first_cursor(statement)))
conn.log.info('%(filename)s:%(lineno)s: %(event)s:\n%(plan)s',
dict(filename=fname, lineno=line, event='plan',
plan=plan))
params = params_used(run_params, statement)
chunk_ix = 0
event_results = conn.execute(statement, params)
while 1:
# self.set_status_message(
# '%s:%s: chunk %d\n%s\n%s\n%s' % (
# fname, line,
# chunk_ix + 1,
# statement, run_params, plan))
with conn.log.step(
'%(filename)s:%(lineno)s: %(event)s %(chunk_num)d',
dict(filename=fname, lineno=line,
event='insert chunk', chunk_num=chunk_ix + 1)) as chunk_step:
event = event_results.fetchone()
if not event:
break
chunk_ix += 1
rowcount = cast(Opt[int], event.row_count) or 0
bulk_rows += rowcount
chunk_step.extra.update(
dict(statement=statement, params=event))
chunk_step.argobj.update(dict(into=event.dest_table,
rowcount=rowcount,
rowsubtotal=bulk_rows))
chunk_step.msg_parts.append(
' %(rowcount)d rows into %(into)s (subtotal: %(rowsubtotal)d)')
return bulk_rows
