def truncate(self, names):
names = list(
filter(lambda name: name in self.materialized_tables, names))
if names:
logger.info('Truncate table in {}: {}'.format(
self.name, ', '.join(names)))
self.sql_helper.truncate(names)
def truncate(self, names):
logger.info('Truncate table in {}: {}'.format(self.name,
', '.join(names)))
for name in names:
try:
path = self.get_path(name)
except ValueError:
continue
with open(path, 'w'):
pass
def execute(self, tasks, task_ids):
task_ids = set(task_ids or ())
for i, task in enumerate(tasks):
if task_ids and i not in task_ids:
continue
self._execute_task(i, task)
logger.info('Finish task {}'.format(i))
self.stores.drop_created_views()
def truncate(self, names):
if names:
logger.info('Truncate table in {}: {}'.format(
self.name, ', '.join(names)))
sql = """
SET FOREIGN_KEY_CHECKS = 0;
{}
SET FOREIGN_KEY_CHECKS = 1;
""".format('\n'.join('TRUNCATE {};'.format(name)
for name in names))
self._execute_sql(sql)
def execute(self, tasks, task_ids):
task_ids = set(task_ids or ())
for i, task in enumerate(tasks):
if task_ids and i not in task_ids:
continue
sources = task.get('from')
dest = task.get('to')
orders = task.pop('orders')
logger.info('Start task {}: Transfer tables from {} to {}'.format(
i, [source['name'] for source in sources], dest['name']))
self._execute_task(sources, dest, orders)
logger.info('Finish task {}'.format(i))
def dispatch(self):
logger.info('Start transfer(PID: {})'.format(os.getpid()))
self._publish()
self._threads_pool.work_queue.join()
# prevent progress bars
printed = False
for task in self._tasks.values():
if type(task) in (RDBToRDBTask, RDBToCSVTask):
printed = True
print()
else:
if printed:
print()
def truncate(self, names):
converted_names = []
for name in names:
try:
name = self._convert_table_name(name)
except ValueError:
# the table not exist in store
pass
else:
converted_names.append(name)
names = list(
filter(lambda name: name in self.materialized_tables,
converted_names))
if names:
logger.info('Truncate table in {}: {}'.format(
self.name, ', '.join(names)))
self.sql_helper.truncate(names)
def _execute_task(self, num, task):
sources = task.get('from')
dest = task.get('to')
orders = task.get('orders')
sourceName = ''
for source in sources:
sourceName = source['name']
logger.info('Start task {}: Transfer tables from {} to {}'.format(
num, [source['name'] for source in sources], dest['name']))
# truncate
tc = TaskClassifier(orders)
effected_tables = tc.effected_tables(sourceName, self.stores.stores)
dest_store = self.stores.find_by_store_name(dest['name'])
dest_store.truncate(effected_tables)
task_dispatcher = TaskDispatcher(self.stores, task)
task_dispatcher.dispatch()
def execute(self, pool=None, watcher=None, consumers_num=3):
logger.info('Execute SQL script in {store}: {name}'.format(
store=self.store.name, name=self.name))
self.store.execute(self.name)
watcher(self.name)
