def register_or_upload_to_pypi(options):
"""
Registers or uploads against a pypi server.
"""
supported_actions = ['register', 'upload']
if len(options) >= 1 and options[0] not in supported_actions:
print('Please provide a supported action (register or upload).')
return
action = options[0]
repository = ''
if len(options) == 2:
# We were provided a explicity repo to target.
# If we were not provided a repo nor does a .pypirc file exists,
# twine will use environment variable TWINE_REPOSITORY, TWINE_USERNAME,
# and TWINE_PASSWORD.
repository = '-r {}'.format(options[1])
print('Repository argument was provided, targeting {}'.format(
options[1]))
mssqlscripter_sdist_name = os.listdir(
utility.MSSQLSCRIPTER_DIST_DIRECTORY)[0]
# Run twine action for mssqlscripter.
utility.exec_command(
'twine {} {} {}'.format(action, mssqlscripter_sdist_name, repository),
utility.MSSQLSCRIPTER_DIST_DIRECTORY)
for wheel_name in os.listdir(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY):
# Run twine action for mssqltoolsservice wheels.
utility.exec_command(
'twine {} {} {}'.format(action, wheel_name, repository),
utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY)
def unit_test():
"""
Run all unit tests.
"""
runid = str(uuid.uuid1())
python_version = platform.python_version()
utility.exec_command(
'pytest --cov mssqlcli --doctest-modules --junitxml=junit/test-{}-results.xml --cov-report=xml --cov-report=html --cov-append '
'-o junit_suite_name=pytest-{} '
'tests/test_mssqlcliclient.py '
'tests/test_completion_refresher.py '
'tests/test_config.py '
'tests/test_naive_completion.py '
'tests/test_main.py '
'tests/test_fuzzy_completion.py '
'tests/test_rowlimit.py '
'tests/test_sqlcompletion.py '
'tests/test_prioritization.py '
'mssqlcli/jsonrpc/tests '
'mssqlcli/jsonrpc/contracts/tests '
'tests/test_telemetry.py '
'tests/test_localization.py '
'tests/test_globalization.py '
'-s tests/test_noninteractive_mode.py ' # -s disables capturing--test doesn't work without it
'tests/test_special.py'.format(runid, python_version),
utility.ROOT_DIR,
continue_on_error=False)
def generate_mo(extraction_target_path,
lang_name,
trans_mappings,
domain,
localedir=None):
"""
Extracts strings from 'extraction_target_path', and creates pot, po, mo file with
'trans_mappings' information.
'extraction_target_path' can be file or directory.
"""
extraction_target_dir = extraction_target_path\
if os.path.isdir(extraction_target_path) else os.path.dirname(extraction_target_path)
localedir = localedir if localedir is not None \
else os.path.join(extraction_target_dir, 'locale')
mo_dir = os.path.join(localedir, lang_name, 'LC_MESSAGES')
create_dir([extraction_target_dir, 'locale', lang_name, 'LC_MESSAGES'])
pot_file = '{0}.pot'.format(os.path.join(localedir, domain))
po_file = '{0}.po'.format(os.path.join(mo_dir, domain))
mo_file = '{0}.mo'.format(os.path.join(mo_dir, domain))
extract_command = "pybabel extract {0} -o {1}".format(
extraction_target_path, pot_file)
utility.exec_command(extract_command, extraction_target_dir)
po = polib.pofile(pot_file)
for entry in po:
if entry.msgid in trans_mappings:
entry.msgstr = trans_mappings[entry.msgid]
po.save(po_file)
po.save_as_mofile(mo_file)
return domain, localedir
def build_sqltoolsservice_wheels(platforms):
"""
For each supported platform, build a universal wheel.
"""
# Clean up dangling directories if previous run was interrupted.
utility.clean_up(directory=TARGET_DIRECTORY)
utility.clean_up(directory=BUILD_DIRECTORY)
if not platforms:
# Defaults to all supported platforms.
platforms = SUPPORTED_PLATFORMS.keys()
print(u'Generating .whl files for the following platforms: {}'.format(
platforms))
for platform in platforms:
if platform not in SUPPORTED_PLATFORMS:
print(u'{} is not a supported platform'.format(platform))
break
# Set environment variable to communicate current platform to setup.py.
os.environ[u'MSSQLTOOLSSERVICE_PLATFORM'] = platform
print(u'Calling setup bdist_wheel for platform:{}'.format(platform))
download_and_unzip(SUPPORTED_PLATFORMS[platform],
directory=TARGET_DIRECTORY)
utility.exec_command(u'python setup.py check -r -s bdist_wheel',
CURRENT_DIRECTORY)
print(
u'Cleaning up mssqltoolservice and build directory for platform:{}'
.format(platform))
utility.clean_up(directory=TARGET_DIRECTORY)
utility.clean_up(directory=BUILD_DIRECTORY)
def unstable_unit_test():
"""
Run all unstable unit tests.
"""
runid = str(uuid.uuid1())
utility.exec_command('pytest --doctest-modules --junitxml=junit/test-unstable-{}-results.xml '
'-s -v -m unstable {}'.format(runid, get_active_test_filepaths()),
utility.ROOT_DIR, continue_on_error=False)
def integration_test():
"""
Run full integration test via tox which includes build, unit tests, code coverage, and packaging.
"""
utility.exec_command(
'tox',
utility.ROOT_DIR,
continue_on_error=False)
def pdf_to_png(self):
"""Convert the PDF file to a SVG file"""
exec_command([
self.checker.inkscape_executable, "--without-gui",
"--pdf-poppler", "--pdf-page=1", "--export-type=png",
"--export-area-drawing", "--export-dpi=300", "--export-file",
self.tmp('png'),
self.tmp('pdf')
])
def unit_test():
"""
Run all unit tests.
"""
utility.exec_command(
'pytest --cov mssqlcli tests/test_mssqlcliclient.py tests/test_main.py tests/test_fuzzy_completion.py '
'tests/test_rowlimit.py tests/test_sqlcompletion.py tests/test_prioritization.py mssqlcli/jsonrpc/contracts/tests '
'tests/test_telemetry.py',
utility.ROOT_DIR,
continue_on_error=False)
def build_wheel_for_current_platform():
"""
Build mssqltoolsservice wheel for current platform.
"""
# Build mssqltoolsservice wheel for this platform.
current_platform = os.environ['MSSQLTOOLSSERVICE_PACKAGE_SUFFIX']
utility.exec_command(
'python mssqltoolsservice/buildwheels.py {}'.format(current_platform),
root_dir,
continue_on_error=False)
def build():
"""
Builds mssql-cli package.
"""
print_heading('Cleanup')
# clean
utility.clean_up(utility.MSSQLCLI_DIST_DIRECTORY)
utility.clean_up_egg_info_sub_directories(utility.ROOT_DIR)
print_heading('Running setup')
# install general requirements.
utility.exec_command(
'{0} -m pip install -r requirements-dev.txt'.format(PYTHON),
utility.ROOT_DIR)
# convert windows line endings to unix for mssql-cli bash script
utility.exec_command('{0} dos2unix.py mssql-cli mssql-cli'.format(PYTHON),
utility.ROOT_DIR)
# run flake8
code_analysis()
if utility.get_current_platform().startswith('win'):
platforms_to_build = ['win32', 'win_amd64']
else:
platforms_to_build = [utility.get_current_platform()]
for plat in platforms_to_build:
# For the current platform, populate the appropriate binaries and
# generate the wheel.
clean_and_copy_sqltoolsservice(plat)
utility.clean_up(utility.MSSQLCLI_BUILD_DIRECTORY)
print_heading('Building mssql-cli pip package')
utility.exec_command('%s --version' % PYTHON, utility.ROOT_DIR)
utility.exec_command('%s setup.py bdist_wheel --plat-name %s' %
(PYTHON, plat),
utility.ROOT_DIR,
continue_on_error=False)
try:
# checks if long description will render correctly--does not work on some systems
utility.exec_command('twine check {}'.format(
os.path.join(utility.MSSQLCLI_DIST_DIRECTORY, '*')),
utility.ROOT_DIR,
continue_on_error=False)
except IOError as err:
print(err)
print("Unable to run 'twine check'.")
# Copy back the SqlToolsService binaries for this platform.
clean_and_copy_sqltoolsservice(utility.get_current_platform())
copy_and_rename_wheels()
def publish_official():
"""
Publish mssql-cli package to PyPi.
"""
mssqlcli_wheel_dir = os.listdir(utility.MSSQLCLI_DIST_DIRECTORY)
# Run twine action for mssql-cli.
# Only authorized users with credentials will be able to upload this package.
# Credentials will be stored in a .pypirc file.
for wheel in mssqlcli_wheel_dir:
utility.exec_command('twine upload {}'.format(wheel),
utility.MSSQLCLI_DIST_DIRECTORY)
def unit_test():
"""
Run all unit tests.
"""
runid = str(uuid.uuid1())
python_version = platform.python_version()
utility.exec_command(('pytest -l --cov mssqlcli --doctest-modules '
'--junitxml=junit/test-{}-results.xml --cov-report=xml '
'--cov-report=html --cov-append -o junit_suite_name=pytest-{} '
'-m "not unstable" {}').format(runid, python_version,
get_active_test_filepaths()),
utility.ROOT_DIR, continue_on_error=False)
def build():
"""
Builds mssql-cli package.
"""
print_heading('Cleanup')
# clean
utility.clean_up(utility.MSSQLCLI_DIST_DIRECTORY)
utility.clean_up_egg_info_sub_directories(utility.ROOT_DIR)
print_heading('Running setup')
# install general requirements.
utility.exec_command('{0} install -r requirements-dev.txt'.format(PIP),
utility.ROOT_DIR)
# convert windows line endings to unix for mssql-cli bash script
utility.exec_command('{0} dos2unix.py mssql-cli mssql-cli'.format(PYTHON),
utility.ROOT_DIR)
# run flake8
code_analysis()
if utility.get_current_platform().startswith('win'):
platforms_to_build = ['win32', 'win_amd64']
else:
platforms_to_build = [utility.get_current_platform()]
for plat in platforms_to_build:
# For the current platform, populate the appropriate binaries and
# generate the wheel.
clean_and_copy_sqltoolsservice(plat)
utility.clean_up(utility.MSSQLCLI_BUILD_DIRECTORY)
print_heading('Building mssql-cli pip package')
utility.exec_command('%s --version' % PYTHON, utility.ROOT_DIR)
utility.exec_command(
'%s setup.py check -r -s bdist_wheel --plat-name %s' %
(PYTHON, plat),
utility.ROOT_DIR,
continue_on_error=False)
# generate sdist--only run on macOS to eliminate redundant copies when published to Azure
# NOTE: macOS was arbitrarily chosen as the single OS
if sys.platform == 'darwin':
utility.exec_command(
'%s setup.py check -r -s sdist --formats=gztar' % PYTHON,
utility.ROOT_DIR,
continue_on_error=False)
# Copy back the SqlToolsService binaries for this platform.
clean_and_copy_sqltoolsservice(utility.get_current_platform())
copy_and_rename_wheels()
def publish_official(platforms_names):
"""
Publish mssql-scripter wheel package to PyPi.
"""
mssqlscripter_wheel_dir = os.listdir(utility.MSSQLSCRIPTER_DIST_DIRECTORY)
# Run twine action for mssqlscripter.
# Only authorized users with credentials will be able to upload this package.
# Credentials will be stored in a .pypirc file.
for mssqlscripter_wheel_name in mssqlscripter_wheel_dir:
utility.exec_command(
'twine upload {}'.format(mssqlscripter_wheel_name),
utility.MSSQLSCRIPTER_DIST_DIRECTORY)
def verify_local_install():
"""
Install mssql-scripter package locally that resolves mssqltoolsservice dependency from local build.
"""
# Local install of mssql-scripter.
mssqlscripter_sdist_name = os.listdir(
utility.MSSQLSCRIPTER_DIST_DIRECTORY)[0]
# To ensure we have a clean install, we disable the cache as to prevent cache overshadowing actual changes made.
utility.exec_command(
'pip install --no-cache-dir --no-index --find-links=./mssqltoolsservice/dist ./dist/{}'
.format(mssqlscripter_sdist_name),
root_dir,
continue_on_error=False)
def tex_to_pdf(self, tex_command, latex_text, preamble_file):
"""
Create a PDF file from latex text
"""
with logger.debug("Converting .tex to .pdf"):
# Read preamble
preamble_file = os.path.abspath(preamble_file)
preamble = ""
if os.path.isfile(preamble_file):
with open(preamble_file, 'r') as f:
preamble += f.read()
# Options pass to LaTeX-related commands
texwrapper = self.DOCUMENT_TEMPLATE % (preamble, latex_text)
# Convert TeX to PDF
# Write tex
with open(self.tmp('tex'), 'w') as f_tex:
f_tex.write(texwrapper)
# Exec tex_command: tex -> pdf
try:
exec_command([tex_command, self.tmp('tex')] +
self.LATEX_OPTIONS)
except TexTextCommandFailed as error:
if os.path.exists(self.tmp('log')):
parsed_log = self.parse_pdf_log(self.tmp('log'))
raise TexTextConversionError(parsed_log,
error.return_code,
error.stdout,
error.stderr)
else:
raise TexTextConversionError(str(error),
error.return_code,
error.stdout,
error.stderr)
if not os.path.exists(self.tmp('pdf')):
raise TexTextConversionError(
"%s didn't produce output %s" %
(tex_command, self.tmp('pdf')))
def validate_package():
"""
Install mssql-cli package locally.
"""
root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), '..'))
# Local install of mssql-scripter.
mssqlcli_wheel_dir = os.listdir(utility.MSSQLCLI_DIST_DIRECTORY)
# To ensure we have a clean install, we disable the cache as to prevent
# cache overshadowing actual changes made.
current_platform = utility.get_current_platform()
mssqlcli_wheel_name = [
pkge for pkge in mssqlcli_wheel_dir if current_platform in pkge
]
utility.exec_command(
'pip install --no-cache-dir --no-index ./dist/{}'.format(
mssqlcli_wheel_name[0]),
root_dir,
continue_on_error=False)
def unit_test():
"""
Run all unit tests.
"""
utility.exec_command(
'pytest --cov mssqlcli --doctest-modules --junitxml=junit/test-results.xml --cov-report=xml --cov-report=html '
'tests/test_mssqlcliclient.py '
'tests/test_completion_refresher.py '
'tests/test_config.py '
'tests/test_naive_completion.py '
'tests/test_main.py '
'tests/test_fuzzy_completion.py '
'tests/test_rowlimit.py '
'tests/test_sqlcompletion.py '
'tests/test_prioritization.py '
'mssqlcli/jsonrpc/tests '
'mssqlcli/jsonrpc/contracts/tests '
'tests/test_telemetry.py '
'tests/test_special.py',
utility.ROOT_DIR,
continue_on_error=False)
def build():
"""
Builds mssql-cli package.
"""
print_heading('Cleanup')
# clean
utility.clean_up(utility.MSSQLCLI_DIST_DIRECTORY)
utility.clean_up_egg_info_sub_directories(utility.ROOT_DIR)
print_heading('Running setup')
# install general requirements.
utility.exec_command('{0} install -r requirements-dev.txt'.format(PIP),
utility.ROOT_DIR)
# convert windows line endings to unix for mssql-cli bash script
utility.exec_command('{0} dos2unix.py mssql-cli mssql-cli'.format(PYTHON),
utility.ROOT_DIR)
# run flake8
code_analysis()
if utility.get_current_platform().startswith('win'):
platforms_to_build = ['win32', 'win_amd64']
else:
platforms_to_build = [utility.get_current_platform()]
for platform in platforms_to_build:
# For the current platform, populate the appropriate binaries and
# generate the wheel.
clean_and_copy_sqltoolsservice(platform)
utility.clean_up(utility.MSSQLCLI_BUILD_DIRECTORY)
print_heading('Building mssql-cli pip package')
utility.exec_command('{0} --version'.format(PYTHON), utility.ROOT_DIR)
utility.exec_command(
'{0} setup.py check -r -s bdist_wheel --plat-name {1}'.format(
PYTHON, platform),
utility.ROOT_DIR,
continue_on_error=False)
# Copy back the SqlToolsService binaries for this platform.
clean_and_copy_sqltoolsservice(utility.get_current_platform())
copy_and_rename_wheels()
def build(platform_names):
"""
Builds mssql-scripter package.
"""
print_heading('Cleanup')
# clean
utility.clean_up(utility.MSSQLSCRIPTER_DIST_DIRECTORY)
print_heading('Running setup')
# install general requirements.
utility.exec_command('pip install -r dev_requirements.txt',
utility.ROOT_DIR)
# convert windows line endings to unix for mssql-cli bash script
utility.exec_command('python dos2unix.py mssql-scripter mssql-scripter',
utility.ROOT_DIR)
for platform in platform_names:
utility.clean_up(utility.MSSQLSCRIPTER_BUILD_DIRECTORY)
utility.cleaun_up_egg_info_sub_directories(utility.ROOT_DIR)
mssqltoolsservice.copy_sqltoolsservice(platform)
print_heading(
'Building mssql-scripter {} wheel package package'.format(
platform))
utility.exec_command('python --version', utility.ROOT_DIR)
utility.exec_command(
'python setup.py check -r -s bdist_wheel --plat-name {}'.format(
platform),
utility.ROOT_DIR,
continue_on_error=False)
mssqltoolsservice.clean_up_sqltoolsservice()
def code_analysis():
utility.exec_command(
'{0} -m flake8 mssqlcli setup.py dev_setup.py build.py utility.py dos2unix.py'.format(PYTHON),
utility.ROOT_DIR)
def code_analysis():
utility.exec_command(
'%s -m flake8 mssqlcli setup.py dev_setup.py build.py utility.py dos2unix.py'
% PYTHON, utility.ROOT_DIR)
#!/usr/bin/env python
from __future__ import print_function
import os
import sys
import utility
PYTHON = os.getenv('CUSTOM_PYTHON', sys.executable)
print('Running dev setup...')
print('Root directory \'%s\'\n' % utility.ROOT_DIR)
# install general requirements.
utility.exec_command('%s -m pip install --no-cache-dir -r requirements-dev.txt' % PYTHON,
utility.ROOT_DIR)
import mssqlcli.mssqltoolsservice.externals as mssqltoolsservice
# download the sqltoolssevice binaries for all platforms
mssqltoolsservice.download_sqltoolsservice_binaries()
# install mssqltoolsservice if this platform supports it.
utility.copy_current_platform_mssqltoolsservice()
print('Finished dev setup.')
#!/usr/bin/env python
# --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------
from __future__ import print_function
import os
import platform
import utility
import mssqlscripter.mssqltoolsservice.external as mssqltoolsservice
print('Running dev setup...')
print('Root directory \'{}\'\n'.format(utility.ROOT_DIR))
# install general requirements.
utility.exec_command('pip install -r dev_requirements.txt', utility.ROOT_DIR)
run_time_id = utility.get_current_platform()
if run_time_id:
mssqltoolsservice.copy_sqltoolsservice(run_time_id)
else:
print("This platform does not support mssqltoolsservice.")
print('Finished dev setup.')
def build(options):
supported_actions = ['nightly']
action = None
if len(options) >= 1:
if options[0] not in supported_actions:
print('Please provide a supported action {}.'.format(
supported_actions))
return
action = options[0]
if action == 'nightly':
assert AZURE_STORAGE_CONNECTION_STRING, 'Set AZURE_STORAGE_CONNECTION_STRING environment variable'
print_heading('Cleanup')
# clean
utility.clean_up(utility.MSSQLSCRIPTER_DIST_DIRECTORY)
utility.clean_up(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY)
utility.cleaun_up_egg_info_sub_directories(utility.ROOT_DIR)
utility.cleaun_up_egg_info_sub_directories(
utility.MSSQLTOOLSSERVICE_DIRECTORY)
print_heading('Running setup')
# install general requirements.
utility.exec_command('pip install -r dev_requirements.txt',
utility.ROOT_DIR)
print_heading('Running mssql-scripter tests')
utility.exec_command('tox', utility.ROOT_DIR, continue_on_error=False)
print_heading('Building mssql-scripter pip package')
utility.exec_command('python setup.py check -r -s sdist',
utility.ROOT_DIR,
continue_on_error=False)
print_heading('Building mssqltoolsservice pip package')
utility.exec_command('python buildwheels.py',
utility.MSSQLTOOLSSERVICE_DIRECTORY,
continue_on_error=False)
if action == 'nightly':
blob_service = BlockBlobService(
connection_string=AZURE_STORAGE_CONNECTION_STRING)
print_heading('Uploading packages to blob storage ')
for pkg in os.listdir(utility.MSSQLSCRIPTER_DIST_DIRECTORY):
pkg_path = os.path.join(utility.MSSQLSCRIPTER_DIST_DIRECTORY, pkg)
print('Uploading package {}'.format(pkg_path))
upload_package(blob_service, pkg_path, 'mssql-scripter')
for pkg in os.listdir(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY):
pkg_path = os.path.join(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY,
pkg)
pkg_name = os.path.basename(pkg_path).split('-')[0].replace(
'_', '-').lower()
print('Uploading package {}'.format(pkg_name))
upload_package(blob_service, pkg_path, pkg_name)
# Upload the final index file
upload_index_file(blob_service, 'index.html', 'Simple Index',
UPLOADED_PACKAGE_LINKS)
#!/usr/bin/env python
from __future__ import print_function
import os
import utility
PIP = os.getenv('CUSTOM_PIP', 'pip')
print('Running dev setup...')
print('Root directory \'%s\'\n' % utility.ROOT_DIR)
# install general requirements.
utility.exec_command('%s install --no-cache-dir -r requirements-dev.txt' % PIP,
utility.ROOT_DIR)
# install mssqltoolsservice if this platform supports it.
utility.copy_current_platform_mssqltoolsservice()
print('Finished dev setup.')
#!/usr/bin/env python
# --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------
from __future__ import print_function
import os
import setup
import utility
root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), '..'))
print('Running dev setup...')
print('Root directory \'{}\'\n'.format(root_dir))
# install general requirements.
utility.exec_command('pip install -r dev_requirements.txt', root_dir)
# install mssqltoolsservice if this platform supports it.
mssqltoolsservice_package_name = os.environ['MSSQLTOOLSSERVICE_PACKAGE_NAME']
print('Installing {}...'.format(mssqltoolsservice_package_name))
# mssqltoolsservice package name is retrieved from environment variable set by setup.py.
utility.exec_command('pip install {}'.format(mssqltoolsservice_package_name),
root_dir)
print('Finished dev setup.')
#!/usr/bin/env python
from __future__ import print_function
import os
import utility
PIP = os.getenv('CUSTOM_PIP', 'pip')
print('Running dev setup...')
print('Root directory \'{}\'\n'.format(utility.ROOT_DIR))
# install general requirements.
utility.exec_command(
'{0} install --no-cache-dir -r requirements-dev.txt'.format(PIP),
utility.ROOT_DIR)
# install mssqltoolsservice if this platform supports it.
utility.copy_current_platform_mssqltoolsservice()
print('Finished dev setup.')
