def create_schemas(items):
"""Create and write schemas from definitions."""
schema_classes, schema_names = create_schemas_classes(items)
items_py = '\n'.join(chain([ITEMS_IMPORTS], schema_classes)).strip()
items_py = fix_lines(fix_code(items_py.decode('utf-8')).splitlines(),
Options)
return items_py, schema_names
def create_schemas(items):
"""Create and write schemas from definitions."""
schema_classes, schema_names = create_schemas_classes(items)
items_py = '\n'.join(chain([ITEMS_IMPORTS], schema_classes)).strip()
items_py = fix_lines(
fix_code(items_py.decode('utf-8')).splitlines(), Options)
return items_py, schema_names
def PEP8():
class Options(object):
aggressive = vim.vars.get('pep8_aggressive', 1)
diff = False
experimental = True
ignore = vim.vars.get('pymode_lint_ignore', ())
in_place = False
indent_size = autopep8.DEFAULT_INDENT_SIZE
line_range = None
max_line_length = int(vim.vars.get('pep8_max_line_length', 78))
pep8_passes = 100
recursive = False
select = vim.vars.get('pymode_lint_select', ())
verbose = 0
start = vim.vvars['lnum'] - 1
end = vim.vvars['lnum'] + vim.vvars['count'] - 1
first_non_blank = int(vim.eval('nextnonblank(v:lnum)')) - 1
last_non_blank = int(vim.eval('prevnonblank(v:lnum + v:count - 1)')) - 1
doc_string = False
if (first_non_blank >= 0
and doc_start.match(vim.current.buffer[first_non_blank])
and doc_end.match(vim.current.buffer[last_non_blank])):
doc_string = True
else:
# Don't remove trailing blank lines except at end of file
while (end < len(vim.current.buffer)
and re.match('^\s*$', vim.current.buffer[end - 1])):
end += 1
lines = vim.current.buffer[start:end]
if not isinstance(lines[0], unicode):
lines = [unicode(line, vim.eval('&encoding') or 'utf-8')
for line in lines]
if doc_string:
new_lines = docformatter.format_code(
u'\n'.join(lines),
force_wrap=bool(vim.vars.get('pep8_force_wrap', 0)),
post_description_blank=False,
pre_summary_newline=True)
else:
options = Options()
if vim.vars['pep8_indent_only']:
options.select = ['E1', 'W1']
new_lines = autopep8.fix_lines(lines, options).lstrip()
new_lines = new_lines.split('\n')[: None if doc_string else -1]
if new_lines != lines:
vim.current.buffer[start:end] = new_lines
def PEP8():
class Options(object):
aggressive = vim.vars.get('pep8_aggressive', 1)
diff = False
experimental = True
ignore = vim.vars.get('pymode_lint_ignore', ())
in_place = False
indent_size = autopep8.DEFAULT_INDENT_SIZE
line_range = None
max_line_length = int(vim.vars.get('pep8_max_line_length', 78))
pep8_passes = 100
recursive = False
select = vim.vars.get('pymode_lint_select', ())
verbose = 0
start = vim.vvars['lnum'] - 1
end = vim.vvars['lnum'] + vim.vvars['count'] - 1
first_non_blank = int(vim.eval('nextnonblank(v:lnum)')) - 1
last_non_blank = int(vim.eval('prevnonblank(v:lnum + v:count - 1)')) - 1
doc_string = False
if (first_non_blank >= 0
and doc_start.match(vim.current.buffer[first_non_blank])
and doc_end.match(vim.current.buffer[last_non_blank])):
doc_string = True
else:
# Don't remove trailing blank lines except at end of file
while (end < len(vim.current.buffer)
and re.match('^\s*$', vim.current.buffer[end - 1])):
end += 1
lines = vim.current.buffer[start:end]
if not isinstance(lines[0], unicode):
lines = [unicode(line, vim.eval('&encoding') or 'utf-8')
for line in lines]
if doc_string:
new_lines = docformatter.format_code(
u'\n'.join(lines),
force_wrap=bool(vim.vars.get('pep8_force_wrap', 0)),
post_description_blank=False,
pre_summary_newline=True)
else:
options = Options()
if vim.vars['pep8_indent_only']:
options.select = ['E1', 'W1']
new_lines = autopep8.fix_lines(lines, options).lstrip()
new_lines = new_lines.split('\n')[: None if doc_string else -1]
if new_lines != lines:
vim.current.buffer[start:end] = new_lines
def create_spiders(spiders, schemas, extractors, items):
"""Create all spiders from slybot spiders."""
item_classes = ''
if items:
item_classes = '\nfrom ..items import {}'.format(', '.join(
(v().__class__.__name__ for v in items.values())))
spider_data = []
for name, (spider, spec) in spiders.items():
log.info('Creating spider "%s"' % spider.name)
spider = create_spider(name, spider, spec, schemas, extractors, items)
cleaned_name = _clean(name)
filename = 'spiders/{}.py'.format(cleaned_name)
data = '\n'.join(
(SPIDER_FILE(item_classes=item_classes), spider.strip()))
code = fix_lines(fix_code(data.decode('utf-8')).splitlines(), Options)
spider_data.append((filename, code))
return spider_data
def create_spiders(spiders, schemas, extractors, items):
"""Create all spiders from slybot spiders."""
item_classes = ''
if items:
item_classes = '\nfrom ..items import {}'.format(
', '.join((v().__class__.__name__ for v in items.values()))
)
spider_data = []
for name, (spider, spec) in spiders.items():
log.info('Creating spider "%s"' % spider.name)
spider = create_spider(name, spider, spec, schemas, extractors, items)
cleaned_name = _clean(name)
filename = 'spiders/{}.py'.format(cleaned_name)
data = '\n'.join((SPIDER_FILE(item_classes=item_classes),
spider.strip()))
code = fix_lines(fix_code(data.decode('utf-8')).splitlines(), Options)
spider_data.append((filename, code))
return spider_data
def fix_code(cells, options):
"""
Returns code lines fixed with autopep8.
"""
autopep8_cmdline = '- ' + options.autopep8_options # Required filename arg
autopep8_options, autopep8_args = autopep8.parse_args(autopep8_cmdline.split())
fixed_cells = []
for cell_num, cell_lines in cells:
fixed_code = autopep8.fix_lines(cell_lines, autopep8_options)
fixed_lines = fixed_code.splitlines(True)
if options.no_newline and fixed_lines:
# Remove the single newline at end of 'file' added by autopep8 to
# each cell.
fixed_lines[-1] = fixed_lines[-1][:-1]
if options.end_semicolon and not fixed_lines[-1].endswith('?'):
fixed_lines[-1] += ';'
fixed_cells.append((cell_num, fixed_lines))
return fixed_cells
except RedisError:
print('Cannot reach redis. Retry...')
time.sleep(.5)
continue
try:
print('Waiting for pep8 jobs...')
while (True):
try:
job = json.loads(redis.brpop('pep8jobs')[1])
except RedisError:
print('Cannot reach redis. Retry...')
time.sleep(.5)
continue
print('Working on job #{}'.format(job['id']))
try:
# list of code lines:
lines = job.pop('lines', None).replace('\r', '').split('\n')
options = autopep8.parse_args([''])
# set of selected errors/warnings:
options.select = {k for k in job.pop('select', None)}
job['result'] = autopep8.fix_lines(lines, options)
except:
traceback.print_exc()
print('Job #{} got canceled!'.format(job['id']))
job['result'] = 'ERROR: canceled job'
finally:
redis.publish('pep8result#{}'.format(job['id']), json.dumps(job))
except KeyboardInterrupt:
pass
def remove_extra_spaces_and_pep8(filename, apply_pep8=True):
"""
removes extra spaces in a filename, replace the file in place
@param filename file name
@param apply_pep8 if True, calls ``autopep8`` on the file
@return number of removed extra spaces
.. versionchanged:: 1.0
Parameter *apply_pep8* was added.
"""
ext = os.path.splitext(filename)[-1]
if ext in (".bat", ".py", ".sh"):
try:
with open(filename, "r") as f:
lines = f.readlines()
encoding = None
except PermissionError as e:
raise PermissionError(filename) from e
except UnicodeDecodeError as e:
try:
with open(filename, "r", encoding="utf-8") as f:
lines = f.readlines()
encoding = "utf-8"
except:
raise Exception(
"unable to load file {} due to unicode errors".format(filename)) from e
else:
try:
with open(filename, "r", encoding="utf-8-sig") as f:
lines = f.readlines()
encoding = "utf-8"
except PermissionError as e:
raise PermissionError(filename) from e
except UnicodeDecodeError as e:
try:
with open(filename, "r") as f:
lines = f.readlines()
encoding = None
except:
raise Exception(
"unable to load file {} due to unicode errors".format(filename)) from e
if len(lines) == 0 and not filename.endswith("__init__.py"):
raise ValueError(
"File '{0}' is empty, encoding='{1}'.".format(filename, encoding))
if ext == ".py":
if encoding is not None and len(lines) > 0 and "#-*-coding:utf-8-*-" in lines[0].replace(" ", ""):
with open(filename, "r", encoding="utf8") as f:
try:
lines = f.readlines()
except UnicodeDecodeError as e:
raise Exception("unable to read: " + filename) from e
encoding = "utf8"
else:
encoding = None
def cdiff(lines):
lines2 = [_.rstrip(" \r\n") for _ in lines]
last = len(lines2) - 1
while last >= 0 and len(lines2[last]) == 0:
last -= 1
last += 1
lines2 = lines2[:last]
diff = len("".join(lines)) - len("\n".join(lines2)) + len(lines)
return diff, lines2
diff, lines2 = cdiff(lines)
ext = os.path.splitext(filename)[-1]
if ext == ".py":
r = autopep8.fix_lines(
lines2,
options=autopep8.parse_args(['']))
if len(lines) > 0 and (len(lines2) == 0 or len(lines2) < len(lines) // 2):
raise ValueError("Resulting file is empty for '{3}',\ninitial number of lines {0} encoding='{1}' diff={2}".format(
len(lines), encoding, diff, filename))
if encoding is None:
with open(filename, "w") as f:
f.write(r)
else:
with open(filename, "w", encoding="utf8") as f:
f.write(r)
if r != "".join(lines):
diff, lines2 = cdiff(r.split("\n"))
else:
diff = 0
elif ext in (".rst", ".md"):
lines2 = [_.replace("\r", "").rstrip("\n ") for _ in lines]
rem = set()
for i, line in enumerate(lines2):
if i >= 1 and lines2[i] == lines2[i - 1] == "":
rem.add(i)
lines2 = [_ for i, _ in enumerate(lines2) if i not in rem]
if len(lines) > 0 and len(lines2[-1]) > 0:
lines2.append("")
if len(lines) > 0 and (len(lines2) == 0 or len(lines2) < len(lines) // 2):
raise ValueError("Resulting file is empty for '{4}',\ninitial number of lines {0} encoding='{1}' len(rem)={2} diff={3}".format(
len(lines), encoding, len(rem), diff, filename))
rl = "".join(lines)
r2 = "\n".join(lines2)
if r2 != rl:
if encoding is None:
with open(filename, "w") as f:
f.write("\n".join(lines2))
else:
with open(filename, "w", encoding="utf8") as f:
f.write("\n".join(lines2))
diff = max(1, diff)
else:
diff = 0
elif diff != 0:
if len(lines) > 0 and (len(lines2) == 0 or len(lines2) < len(lines) // 2):
raise ValueError("Resulting file is empty for '{3}',\ninitial number of lines {0} encoding='{1}' diff={2}".format(
len(lines), encoding, diff, filename))
r1 = "".join(lines)
r2 = "\n".join(lines2)
if r2 != r1:
if encoding is None:
with open(filename, "w") as f:
f.write("\n".join(lines2))
else:
with open(filename, "w", encoding="utf8") as f:
f.write("\n".join(lines2))
if not os.path.exists(filename):
raise FileNotFoundError(
"issue when applying autopep8 with filename: {0}".format(filename))
return diff
