def truncate(self, names): names = list( filter(lambda name: name in self.materialized_tables, names)) if names: logger.info('Truncate table in {}: {}'.format( self.name, ', '.join(names))) self.sql_helper.truncate(names)
def truncate(self, names): logger.info('Truncate table in {}: {}'.format(self.name, ', '.join(names))) for name in names: try: path = self.get_path(name) except ValueError: continue with open(path, 'w'): pass
def execute(self, tasks, task_ids): task_ids = set(task_ids or ()) for i, task in enumerate(tasks): if task_ids and i not in task_ids: continue self._execute_task(i, task) logger.info('Finish task {}'.format(i)) self.stores.drop_created_views()
def truncate(self, names): if names: logger.info('Truncate table in {}: {}'.format( self.name, ', '.join(names))) sql = """ SET FOREIGN_KEY_CHECKS = 0; {} SET FOREIGN_KEY_CHECKS = 1; """.format('\n'.join('TRUNCATE {};'.format(name) for name in names)) self._execute_sql(sql)
def execute(self, tasks, task_ids): task_ids = set(task_ids or ()) for i, task in enumerate(tasks): if task_ids and i not in task_ids: continue sources = task.get('from') dest = task.get('to') orders = task.pop('orders') logger.info('Start task {}: Transfer tables from {} to {}'.format( i, [source['name'] for source in sources], dest['name'])) self._execute_task(sources, dest, orders) logger.info('Finish task {}'.format(i))
def dispatch(self): logger.info('Start transfer(PID: {})'.format(os.getpid())) self._publish() self._threads_pool.work_queue.join() # prevent progress bars printed = False for task in self._tasks.values(): if type(task) in (RDBToRDBTask, RDBToCSVTask): printed = True print() else: if printed: print()
def truncate(self, names): converted_names = [] for name in names: try: name = self._convert_table_name(name) except ValueError: # the table not exist in store pass else: converted_names.append(name) names = list( filter(lambda name: name in self.materialized_tables, converted_names)) if names: logger.info('Truncate table in {}: {}'.format( self.name, ', '.join(names))) self.sql_helper.truncate(names)
def _execute_task(self, num, task): sources = task.get('from') dest = task.get('to') orders = task.get('orders') sourceName = '' for source in sources: sourceName = source['name'] logger.info('Start task {}: Transfer tables from {} to {}'.format( num, [source['name'] for source in sources], dest['name'])) # truncate tc = TaskClassifier(orders) effected_tables = tc.effected_tables(sourceName, self.stores.stores) dest_store = self.stores.find_by_store_name(dest['name']) dest_store.truncate(effected_tables) task_dispatcher = TaskDispatcher(self.stores, task) task_dispatcher.dispatch()
def execute(self, pool=None, watcher=None, consumers_num=3): logger.info('Execute SQL script in {store}: {name}'.format( store=self.store.name, name=self.name)) self.store.execute(self.name) watcher(self.name)