def gen(df):
size = len(df)
bar = init_progress_bar(size)
try:
c = itertools.count()
for row in df.values:
i = next(c)
if i % 50 == 0:
bar.update(min(i, size))
yield tb.new_record(list(row))
bar.update(size)
finally:
bar.close()
def gen(df):
size = len(df)
bar = init_progress_bar(size)
try:
c = itertools.count()
for row in df.values:
i = next(c)
if i % 50 == 0:
bar.update(min(i, size))
yield tb.new_record(list(row))
bar.update(size)
finally:
bar.close()
def execute(self, line, cell=""):
self._set_odps()
sql = line + "\n" + cell
sql = sql.strip()
if sql:
bar = init_progress_bar()
instance = self._odps.run_sql(sql)
percent = 0
while not instance.is_terminated():
task_names = instance.get_task_names()
last_percent = percent
if len(task_names) > 0:
percent = sum(self._get_task_percent(instance, name) for name in task_names) / len(task_names)
else:
percent = 0
percent = min(1, max(percent, last_percent))
bar.update(percent)
time.sleep(1)
instance.wait_for_success()
bar.update(1)
try:
with instance.open_reader() as reader:
try:
import pandas as pd
try:
return pd.read_csv(StringIO(reader.raw))
except ValueError:
return reader.raw
except ImportError:
return ResultFrame(list(reader), columns=reader._columns)
finally:
bar.close()
def execute(self, line, cell=''):
if self._odps is None:
self._odps = enter().odps
sql = line + '\n' + cell
sql = sql.strip()
if sql:
bar = init_progress_bar()
instance = self._odps.run_sql(sql)
percent = 0
while not instance.is_terminated():
task_names = instance.get_task_names()
last_percent = percent
if len(task_names) > 0:
percent = sum(
self._get_task_percent(instance, name)
for name in task_names) / len(task_names)
else:
percent = 0
percent = min(1, max(percent, last_percent))
bar.update(percent)
time.sleep(1)
instance.wait_for_success()
bar.update(1)
with instance.open_reader() as reader:
try:
import pandas as pd
try:
return pd.read_csv(StringIO(reader.raw))
except ValueError:
return reader.raw
except ImportError:
return ResultFrame(list(reader), columns=reader._columns)
def execute(self, line, cell=''):
self._set_odps()
content = line + '\n' + cell
content = content.strip()
sql = None
hints = dict()
splits = content.split(';')
for s in splits:
stripped = s.strip()
if stripped.lower().startswith('set '):
hint = stripped.split(' ', 1)[1]
k, v = hint.split('=', 1)
k, v = k.strip(), v.strip()
hints[k] = v
elif len(stripped) == 0:
continue
else:
if sql is None:
sql = s
else:
sql = '%s;%s' % (sql, s)
# replace user defined parameters
sql = replace_sql_parameters(sql, self.shell.user_ns)
if sql:
bar = init_progress_bar()
instance = self._odps.run_sql(sql, hints=hints)
if options.verbose:
stdout = options.verbose_log or self._to_stdout
stdout('Instance ID: ' + instance.id)
stdout(' Log view: ' + instance.get_logview_address())
percent = 0
while not instance.is_terminated():
task_names = instance.get_task_names()
last_percent = percent
if len(task_names) > 0:
percent = sum(self._get_task_percent(instance, name)
for name in task_names) / len(task_names)
else:
percent = 0
percent = min(1, max(percent, last_percent))
bar.update(percent)
time.sleep(1)
instance.wait_for_success()
bar.update(1)
try:
with instance.open_reader() as reader:
try:
import pandas as pd
from pandas.parser import CParserError
try:
res = pd.read_csv(StringIO(reader.raw))
except (ValueError, CParserError):
res = reader.raw
except ImportError:
try:
res = ResultFrame(list(reader), columns=reader._columns)
except TypeError:
res = reader.raw
html_notify('SQL execution succeeded')
return res
finally:
bar.close()
def execute(self, line, cell=''):
self._set_odps()
content = line + '\n' + cell
content = content.strip()
sql = None
hints = dict()
splits = content.split(';')
for s in splits:
stripped = s.strip()
if stripped.lower().startswith('set '):
hint = stripped.split(' ', 1)[1]
k, v = hint.split('=', 1)
k, v = k.strip(), v.strip()
hints[k] = v
elif len(stripped) == 0:
continue
else:
if sql is None:
sql = s
else:
sql = '%s;%s' % (sql, s)
# replace user defined parameters
sql = replace_sql_parameters(sql, self.shell.user_ns)
if sql:
bar = init_progress_bar()
instance = self._odps.run_sql(sql, hints=hints)
if options.verbose:
stdout = options.verbose_log or self._to_stdout
stdout('Instance ID: ' + instance.id)
stdout(' Log view: ' + instance.get_logview_address())
percent = 0
while not instance.is_terminated():
task_names = instance.get_task_names()
last_percent = percent
if len(task_names) > 0:
percent = sum(self._get_task_percent(instance, name)
for name in task_names) / len(task_names)
else:
percent = 0
percent = min(1, max(percent, last_percent))
bar.update(percent)
time.sleep(1)
instance.wait_for_success()
bar.update(1)
try:
with instance.open_reader() as reader:
try:
import pandas as pd
from pandas.parser import CParserError
try:
return pd.read_csv(StringIO(reader.raw))
except (ValueError, CParserError):
return reader.raw
except ImportError:
try:
return ResultFrame(list(reader), columns=reader._columns)
except TypeError:
return reader.raw
finally:
bar.close()
