def gen(df): size = len(df) bar = init_progress_bar(size) try: c = itertools.count() for row in df.values: i = next(c) if i % 50 == 0: bar.update(min(i, size)) yield tb.new_record(list(row)) bar.update(size) finally: bar.close()
def execute(self, line, cell=""): self._set_odps() sql = line + "\n" + cell sql = sql.strip() if sql: bar = init_progress_bar() instance = self._odps.run_sql(sql) percent = 0 while not instance.is_terminated(): task_names = instance.get_task_names() last_percent = percent if len(task_names) > 0: percent = sum(self._get_task_percent(instance, name) for name in task_names) / len(task_names) else: percent = 0 percent = min(1, max(percent, last_percent)) bar.update(percent) time.sleep(1) instance.wait_for_success() bar.update(1) try: with instance.open_reader() as reader: try: import pandas as pd try: return pd.read_csv(StringIO(reader.raw)) except ValueError: return reader.raw except ImportError: return ResultFrame(list(reader), columns=reader._columns) finally: bar.close()
def execute(self, line, cell=''): if self._odps is None: self._odps = enter().odps sql = line + '\n' + cell sql = sql.strip() if sql: bar = init_progress_bar() instance = self._odps.run_sql(sql) percent = 0 while not instance.is_terminated(): task_names = instance.get_task_names() last_percent = percent if len(task_names) > 0: percent = sum( self._get_task_percent(instance, name) for name in task_names) / len(task_names) else: percent = 0 percent = min(1, max(percent, last_percent)) bar.update(percent) time.sleep(1) instance.wait_for_success() bar.update(1) with instance.open_reader() as reader: try: import pandas as pd try: return pd.read_csv(StringIO(reader.raw)) except ValueError: return reader.raw except ImportError: return ResultFrame(list(reader), columns=reader._columns)
def execute(self, line, cell=''): self._set_odps() content = line + '\n' + cell content = content.strip() sql = None hints = dict() splits = content.split(';') for s in splits: stripped = s.strip() if stripped.lower().startswith('set '): hint = stripped.split(' ', 1)[1] k, v = hint.split('=', 1) k, v = k.strip(), v.strip() hints[k] = v elif len(stripped) == 0: continue else: if sql is None: sql = s else: sql = '%s;%s' % (sql, s) # replace user defined parameters sql = replace_sql_parameters(sql, self.shell.user_ns) if sql: bar = init_progress_bar() instance = self._odps.run_sql(sql, hints=hints) if options.verbose: stdout = options.verbose_log or self._to_stdout stdout('Instance ID: ' + instance.id) stdout(' Log view: ' + instance.get_logview_address()) percent = 0 while not instance.is_terminated(): task_names = instance.get_task_names() last_percent = percent if len(task_names) > 0: percent = sum(self._get_task_percent(instance, name) for name in task_names) / len(task_names) else: percent = 0 percent = min(1, max(percent, last_percent)) bar.update(percent) time.sleep(1) instance.wait_for_success() bar.update(1) try: with instance.open_reader() as reader: try: import pandas as pd from pandas.parser import CParserError try: res = pd.read_csv(StringIO(reader.raw)) except (ValueError, CParserError): res = reader.raw except ImportError: try: res = ResultFrame(list(reader), columns=reader._columns) except TypeError: res = reader.raw html_notify('SQL execution succeeded') return res finally: bar.close()
def execute(self, line, cell=''): self._set_odps() content = line + '\n' + cell content = content.strip() sql = None hints = dict() splits = content.split(';') for s in splits: stripped = s.strip() if stripped.lower().startswith('set '): hint = stripped.split(' ', 1)[1] k, v = hint.split('=', 1) k, v = k.strip(), v.strip() hints[k] = v elif len(stripped) == 0: continue else: if sql is None: sql = s else: sql = '%s;%s' % (sql, s) # replace user defined parameters sql = replace_sql_parameters(sql, self.shell.user_ns) if sql: bar = init_progress_bar() instance = self._odps.run_sql(sql, hints=hints) if options.verbose: stdout = options.verbose_log or self._to_stdout stdout('Instance ID: ' + instance.id) stdout(' Log view: ' + instance.get_logview_address()) percent = 0 while not instance.is_terminated(): task_names = instance.get_task_names() last_percent = percent if len(task_names) > 0: percent = sum(self._get_task_percent(instance, name) for name in task_names) / len(task_names) else: percent = 0 percent = min(1, max(percent, last_percent)) bar.update(percent) time.sleep(1) instance.wait_for_success() bar.update(1) try: with instance.open_reader() as reader: try: import pandas as pd from pandas.parser import CParserError try: return pd.read_csv(StringIO(reader.raw)) except (ValueError, CParserError): return reader.raw except ImportError: try: return ResultFrame(list(reader), columns=reader._columns) except TypeError: return reader.raw finally: bar.close()