def _run_linter(
args: argparse.Namespace, linter: linters.Linter,
filenames: Sequence[Text], validate_only: bool,
diagnostics_output: DiagnosticsOutput) -> Tuple[Set[Text], bool]:
'''Runs the linter against all files.'''
logging.debug('%s: Files to consider: %s', linter.name,
' '.join(filenames))
logging.debug('%s: Running with %d threads', linter.name, args.jobs)
files = dict((filename, git_tools.file_contents(args, _ROOT, filename))
for filename in filenames)
with concurrent.futures.ThreadPoolExecutor(
max_workers=args.jobs) as executor:
futures = []
for filename, contents in files.items():
futures.append(
executor.submit(_run_linter_one, linter, filename, contents,
validate_only, diagnostics_output))
results = [
f.result() for f in concurrent.futures.as_completed(futures)
]
results.extend(
_run_linter_all(args, linter, filenames, validate_only,
diagnostics_output))
return (set(violation for violation, _ in results
if violation is not None),
any(fixable for _, fixable in results))
def main():
'''Runs the linters against the chosen files.'''
args = git_tools.parse_arguments(
tool_description='validates schema.sql',
extra_arguments=[
git_tools.Argument(
'--mysql-config-file',
default=database_utils.default_config_file(),
help='.my.cnf file that stores credentials'),
git_tools.Argument(
'--database', default='omegaup', help='MySQL database'),
git_tools.Argument(
'--username', default='root', help='MySQL root username'),
git_tools.Argument(
'--password', default='omegaup', help='MySQL password')])
# If running in an automated environment, we can close stdin.
# This will disable all prompts.
if (args.continuous_integration
or os.environ.get('CONTINUOUS_INTEGRATION') == 'true'):
sys.stdin.close()
validate_only = args.tool == 'validate'
filtered_files = list(filename for filename in args.files if
filename.endswith('.sql'))
if not filtered_files:
return
root = git_tools.root_dir()
expected = _expected_database_schema(config_file=args.mysql_config_file,
username=args.username,
password=args.password,
verbose=args.verbose)
actual = git_tools.file_contents(
args, root, 'frontend/database/schema.sql')
if (strip_mysql_extensions(expected.strip()) != strip_mysql_extensions(
actual.strip())):
if validate_only:
if git_tools.attempt_automatic_fixes(sys.argv[0], args,
filtered_files):
sys.exit(1)
print('%sschema.sql validation errors.%s '
'Please run `%s` to fix them.' % (
git_tools.COLORS.FAIL, git_tools.COLORS.NORMAL,
git_tools.get_fix_commandline(sys.argv[0], args,
filtered_files)),
file=sys.stderr)
else:
with open(os.path.join(root,
'frontend/database/schema.sql'), 'wb') as f:
f.write(expected)
print('Files written to working directory. '
'%sPlease commit them before pushing.%s' % (
git_tools.COLORS.HEADER, git_tools.COLORS.NORMAL),
file=sys.stderr)
sys.exit(1)
def _run_linter_all(
args: argparse.Namespace, linter: linters.Linter, files: Sequence[Text],
validate_only: bool, diagnostics_output: DiagnosticsOutput
) -> Sequence[Tuple[Optional[Text], bool]]:
try:
try:
new_file_contents, original_contents, violations = linter.run_all(
files, lambda filename: git_tools.file_contents(
args, _ROOT, filename))
except linters.LinterException:
raise
except: # noqa: bare-except
exc_type, exc_value, exc_traceback = sys.exc_info()
raise linters.LinterException(
'Linter error',
fixable=False,
diagnostics=[
linters.Diagnostic(
message=''.join(
traceback.format_exception(exc_type, exc_value,
exc_traceback)),
filename=', '.join(files),
),
],
) from None
except linters.LinterException as lex:
_report_linter_exception(', '.join(files), lex, diagnostics_output)
return [(filename, lex.fixable) for filename in files]
result: List[Tuple[Optional[Text], bool]] = []
for filename in new_file_contents:
if original_contents[filename] == new_file_contents[filename]:
result.append((None, False))
else:
result.append(
_report_linter_results(filename, new_file_contents[filename],
validate_only, violations, True))
return result
def _run_linter_all(
args: argparse.Namespace, linter: linters.Linter,
files: Sequence[Text], validate_only: bool,
diagnostics_output: DiagnosticsOutput
) -> Sequence[Tuple[Optional[Text], bool]]:
try:
new_file_contents, original_contents, violations = linter.run_all(
files,
lambda filename: git_tools.file_contents(args, _ROOT, filename))
except linters.LinterException as lex:
_report_linter_exception(', '.join(files), lex, diagnostics_output)
return [(filename, lex.fixable) for filename in files]
result: List[Tuple[Optional[Text], bool]] = []
for filename in new_file_contents:
if original_contents[filename] == new_file_contents[filename]:
result.append((None, False))
else:
result.append(_report_linter_results(filename,
new_file_contents[filename],
validate_only, violations,
True))
return result
def main():
'''Runs the linters against the chosen files.'''
args = git_tools.parse_arguments(
tool_description='validates schema.sql',
extra_arguments=[
git_tools.Argument('--mysql-config-file',
default=database_utils.default_config_file(),
help='.my.cnf file that stores credentials'),
git_tools.Argument('--database',
default='omegaup',
help='MySQL database'),
git_tools.Argument('--username',
default='root',
help='MySQL root username'),
git_tools.Argument('--password',
default='omegaup',
help='MySQL password')
])
# If running in an automated environment, we can close stdin.
# This will disable all prompts.
if (args.continuous_integration
or os.environ.get('CONTINUOUS_INTEGRATION') == 'true'):
sys.stdin.close()
validate_only = args.tool == 'validate'
filtered_files = list(filename for filename in args.files
if filename.endswith('.sql'))
if not filtered_files:
return
root = git_tools.root_dir()
expected = _expected_database_schema(config_file=args.mysql_config_file,
username=args.username,
password=args.password,
verbose=args.verbose)
actual = git_tools.file_contents(args, root,
'frontend/database/schema.sql')
if (strip_mysql_extensions(expected.strip()) != strip_mysql_extensions(
actual.strip())):
if validate_only:
if git_tools.attempt_automatic_fixes(sys.argv[0], args,
filtered_files):
sys.exit(1)
print('%sschema.sql validation errors.%s '
'Please run `%s` to fix them.' %
(git_tools.COLORS.FAIL, git_tools.COLORS.NORMAL,
git_tools.get_fix_commandline(sys.argv[0], args,
filtered_files)),
file=sys.stderr)
else:
with open(os.path.join(root, 'frontend/database/schema.sql'),
'wb') as f:
f.write(expected)
print('Files written to working directory. '
'%sPlease commit them before pushing.%s' %
(git_tools.COLORS.HEADER, git_tools.COLORS.NORMAL),
file=sys.stderr)
sys.exit(1)
def _check_mutually_exclusive_schema_modifications(
*,
args: argparse.Namespace,
root: str,
) -> bool:
'''Ensures that schema.sql and dao_schema.sql are not modified together.'''
merge_base = subprocess.run(
[
'/usr/bin/git',
'rev-parse',
'--abbrev-ref',
'--symbolic-full-name',
'@{u}',
],
check=False,
universal_newlines=True,
stdout=subprocess.PIPE,
cwd=root,
).stdout.strip() or 'origin/main'
modified_files = set(
filename.decode('utf-8') for filename in subprocess.run(
[
'/usr/bin/git',
'--no-pager',
'diff',
'-z',
'--name-only',
merge_base,
],
check=True,
stdout=subprocess.PIPE,
cwd=root).stdout.strip(b'\x00').split(b'\x00'))
schema_sql_filename = 'frontend/database/schema.sql'
dao_schema_sql_filename = 'frontend/database/dao_schema.sql'
schema_sql_modified = schema_sql_filename in modified_files
dao_schema_sql_modified = dao_schema_sql_filename in modified_files
if not schema_sql_modified and not dao_schema_sql_modified:
# Neither file got modified, all's good.
return True
if schema_sql_modified and dao_schema_sql_modified:
# Welp, both files got modified, this is bad.
print('%s%r and %r cannot be modified in the same commit.%s' %
(git_tools.COLORS.FAIL, schema_sql_filename,
dao_schema_sql_filename, git_tools.COLORS.NORMAL),
file=sys.stderr)
return False
if schema_sql_modified:
# This is okay. Only the schema.sql file was modified. The rest of this
# script will validate whether it has the correct contents.
return True
schema_sql = git_tools.file_contents(args, root, schema_sql_filename)
dao_schema_sql = git_tools.file_contents(args, root,
dao_schema_sql_filename)
if schema_sql != dao_schema_sql:
print('%s%r can only have the same contents as %r.%s' %
(git_tools.COLORS.FAIL, dao_schema_sql_filename,
schema_sql_filename, git_tools.COLORS.NORMAL),
file=sys.stderr)
return False
return True
def main() -> None:
'''Runs the linters against the chosen files.'''
args = git_tools.parse_arguments(
tool_description='validates schema.sql',
extra_arguments=[
git_tools.Argument('--mysql-config-file',
default=database_utils.default_config_file(),
help='.my.cnf file that stores credentials'),
git_tools.Argument('--database',
default='omegaup',
help='MySQL database'),
git_tools.Argument('--hostname',
default=None,
type=str,
help='Hostname of the MySQL server'),
git_tools.Argument('--username',
default='root',
help='MySQL root username'),
git_tools.Argument('--password',
default='omegaup',
help='MySQL password')
])
validate_only = args.tool == 'validate'
filtered_files = list(filename for filename in args.files
if filename.endswith('.sql'))
root = git_tools.root_dir()
if not _check_mutually_exclusive_schema_modifications(
args=args,
root=root,
):
sys.exit(1)
if 'frontend/database/dao_schema.sql' in filtered_files:
filtered_files.remove('frontend/database/dao_schema.sql')
if not filtered_files:
return
expected = _expected_database_schema(config_file=args.mysql_config_file,
username=args.username,
password=args.password,
hostname=args.hostname,
verbose=args.verbose)
actual = git_tools.file_contents(args, root, _SCHEMA_FILENAME)
expected_contents = strip_mysql_extensions(expected.strip())
actual_contents = strip_mysql_extensions(actual.strip())
if expected_contents != actual_contents:
if validate_only:
if git_tools.attempt_automatic_fixes(sys.argv[0], args,
filtered_files):
sys.exit(1)
sys.stderr.writelines(
difflib.unified_diff(
actual_contents.decode('utf-8').splitlines(keepends=True),
expected_contents.decode('utf-8').splitlines(
keepends=True),
fromfile=_SCHEMA_FILENAME,
tofile=_SCHEMA_FILENAME))
print('%sschema.sql validation errors.%s '
'Please run `%s` to fix them.' %
(git_tools.COLORS.FAIL, git_tools.COLORS.NORMAL,
git_tools.get_fix_commandline(args, filtered_files)),
file=sys.stderr)
else:
with open(os.path.join(root, 'frontend/database/schema.sql'),
'wb') as f:
f.write(expected)
print('Files written to working directory. '
'%sPlease commit them before pushing.%s' %
(git_tools.COLORS.HEADER, git_tools.COLORS.NORMAL),
file=sys.stderr)
sys.exit(1)