def register_or_upload_to_pypi(options): """ Registers or uploads against a pypi server. """ supported_actions = ['register', 'upload'] if len(options) >= 1 and options[0] not in supported_actions: print('Please provide a supported action (register or upload).') return action = options[0] repository = '' if len(options) == 2: # We were provided a explicity repo to target. # If we were not provided a repo nor does a .pypirc file exists, # twine will use environment variable TWINE_REPOSITORY, TWINE_USERNAME, # and TWINE_PASSWORD. repository = '-r {}'.format(options[1]) print('Repository argument was provided, targeting {}'.format( options[1])) mssqlscripter_sdist_name = os.listdir( utility.MSSQLSCRIPTER_DIST_DIRECTORY)[0] # Run twine action for mssqlscripter. utility.exec_command( 'twine {} {} {}'.format(action, mssqlscripter_sdist_name, repository), utility.MSSQLSCRIPTER_DIST_DIRECTORY) for wheel_name in os.listdir(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY): # Run twine action for mssqltoolsservice wheels. utility.exec_command( 'twine {} {} {}'.format(action, wheel_name, repository), utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY)
def unit_test(): """ Run all unit tests. """ runid = str(uuid.uuid1()) python_version = platform.python_version() utility.exec_command( 'pytest --cov mssqlcli --doctest-modules --junitxml=junit/test-{}-results.xml --cov-report=xml --cov-report=html --cov-append ' '-o junit_suite_name=pytest-{} ' 'tests/test_mssqlcliclient.py ' 'tests/test_completion_refresher.py ' 'tests/test_config.py ' 'tests/test_naive_completion.py ' 'tests/test_main.py ' 'tests/test_fuzzy_completion.py ' 'tests/test_rowlimit.py ' 'tests/test_sqlcompletion.py ' 'tests/test_prioritization.py ' 'mssqlcli/jsonrpc/tests ' 'mssqlcli/jsonrpc/contracts/tests ' 'tests/test_telemetry.py ' 'tests/test_localization.py ' 'tests/test_globalization.py ' '-s tests/test_noninteractive_mode.py ' # -s disables capturing--test doesn't work without it 'tests/test_special.py'.format(runid, python_version), utility.ROOT_DIR, continue_on_error=False)
def generate_mo(extraction_target_path, lang_name, trans_mappings, domain, localedir=None): """ Extracts strings from 'extraction_target_path', and creates pot, po, mo file with 'trans_mappings' information. 'extraction_target_path' can be file or directory. """ extraction_target_dir = extraction_target_path\ if os.path.isdir(extraction_target_path) else os.path.dirname(extraction_target_path) localedir = localedir if localedir is not None \ else os.path.join(extraction_target_dir, 'locale') mo_dir = os.path.join(localedir, lang_name, 'LC_MESSAGES') create_dir([extraction_target_dir, 'locale', lang_name, 'LC_MESSAGES']) pot_file = '{0}.pot'.format(os.path.join(localedir, domain)) po_file = '{0}.po'.format(os.path.join(mo_dir, domain)) mo_file = '{0}.mo'.format(os.path.join(mo_dir, domain)) extract_command = "pybabel extract {0} -o {1}".format( extraction_target_path, pot_file) utility.exec_command(extract_command, extraction_target_dir) po = polib.pofile(pot_file) for entry in po: if entry.msgid in trans_mappings: entry.msgstr = trans_mappings[entry.msgid] po.save(po_file) po.save_as_mofile(mo_file) return domain, localedir
def build_sqltoolsservice_wheels(platforms): """ For each supported platform, build a universal wheel. """ # Clean up dangling directories if previous run was interrupted. utility.clean_up(directory=TARGET_DIRECTORY) utility.clean_up(directory=BUILD_DIRECTORY) if not platforms: # Defaults to all supported platforms. platforms = SUPPORTED_PLATFORMS.keys() print(u'Generating .whl files for the following platforms: {}'.format( platforms)) for platform in platforms: if platform not in SUPPORTED_PLATFORMS: print(u'{} is not a supported platform'.format(platform)) break # Set environment variable to communicate current platform to setup.py. os.environ[u'MSSQLTOOLSSERVICE_PLATFORM'] = platform print(u'Calling setup bdist_wheel for platform:{}'.format(platform)) download_and_unzip(SUPPORTED_PLATFORMS[platform], directory=TARGET_DIRECTORY) utility.exec_command(u'python setup.py check -r -s bdist_wheel', CURRENT_DIRECTORY) print( u'Cleaning up mssqltoolservice and build directory for platform:{}' .format(platform)) utility.clean_up(directory=TARGET_DIRECTORY) utility.clean_up(directory=BUILD_DIRECTORY)
def unstable_unit_test(): """ Run all unstable unit tests. """ runid = str(uuid.uuid1()) utility.exec_command('pytest --doctest-modules --junitxml=junit/test-unstable-{}-results.xml ' '-s -v -m unstable {}'.format(runid, get_active_test_filepaths()), utility.ROOT_DIR, continue_on_error=False)
def integration_test(): """ Run full integration test via tox which includes build, unit tests, code coverage, and packaging. """ utility.exec_command( 'tox', utility.ROOT_DIR, continue_on_error=False)
def pdf_to_png(self): """Convert the PDF file to a SVG file""" exec_command([ self.checker.inkscape_executable, "--without-gui", "--pdf-poppler", "--pdf-page=1", "--export-type=png", "--export-area-drawing", "--export-dpi=300", "--export-file", self.tmp('png'), self.tmp('pdf') ])
def unit_test(): """ Run all unit tests. """ utility.exec_command( 'pytest --cov mssqlcli tests/test_mssqlcliclient.py tests/test_main.py tests/test_fuzzy_completion.py ' 'tests/test_rowlimit.py tests/test_sqlcompletion.py tests/test_prioritization.py mssqlcli/jsonrpc/contracts/tests ' 'tests/test_telemetry.py', utility.ROOT_DIR, continue_on_error=False)
def build_wheel_for_current_platform(): """ Build mssqltoolsservice wheel for current platform. """ # Build mssqltoolsservice wheel for this platform. current_platform = os.environ['MSSQLTOOLSSERVICE_PACKAGE_SUFFIX'] utility.exec_command( 'python mssqltoolsservice/buildwheels.py {}'.format(current_platform), root_dir, continue_on_error=False)
def build(): """ Builds mssql-cli package. """ print_heading('Cleanup') # clean utility.clean_up(utility.MSSQLCLI_DIST_DIRECTORY) utility.clean_up_egg_info_sub_directories(utility.ROOT_DIR) print_heading('Running setup') # install general requirements. utility.exec_command( '{0} -m pip install -r requirements-dev.txt'.format(PYTHON), utility.ROOT_DIR) # convert windows line endings to unix for mssql-cli bash script utility.exec_command('{0} dos2unix.py mssql-cli mssql-cli'.format(PYTHON), utility.ROOT_DIR) # run flake8 code_analysis() if utility.get_current_platform().startswith('win'): platforms_to_build = ['win32', 'win_amd64'] else: platforms_to_build = [utility.get_current_platform()] for plat in platforms_to_build: # For the current platform, populate the appropriate binaries and # generate the wheel. clean_and_copy_sqltoolsservice(plat) utility.clean_up(utility.MSSQLCLI_BUILD_DIRECTORY) print_heading('Building mssql-cli pip package') utility.exec_command('%s --version' % PYTHON, utility.ROOT_DIR) utility.exec_command('%s setup.py bdist_wheel --plat-name %s' % (PYTHON, plat), utility.ROOT_DIR, continue_on_error=False) try: # checks if long description will render correctly--does not work on some systems utility.exec_command('twine check {}'.format( os.path.join(utility.MSSQLCLI_DIST_DIRECTORY, '*')), utility.ROOT_DIR, continue_on_error=False) except IOError as err: print(err) print("Unable to run 'twine check'.") # Copy back the SqlToolsService binaries for this platform. clean_and_copy_sqltoolsservice(utility.get_current_platform()) copy_and_rename_wheels()
def publish_official(): """ Publish mssql-cli package to PyPi. """ mssqlcli_wheel_dir = os.listdir(utility.MSSQLCLI_DIST_DIRECTORY) # Run twine action for mssql-cli. # Only authorized users with credentials will be able to upload this package. # Credentials will be stored in a .pypirc file. for wheel in mssqlcli_wheel_dir: utility.exec_command('twine upload {}'.format(wheel), utility.MSSQLCLI_DIST_DIRECTORY)
def unit_test(): """ Run all unit tests. """ runid = str(uuid.uuid1()) python_version = platform.python_version() utility.exec_command(('pytest -l --cov mssqlcli --doctest-modules ' '--junitxml=junit/test-{}-results.xml --cov-report=xml ' '--cov-report=html --cov-append -o junit_suite_name=pytest-{} ' '-m "not unstable" {}').format(runid, python_version, get_active_test_filepaths()), utility.ROOT_DIR, continue_on_error=False)
def build(): """ Builds mssql-cli package. """ print_heading('Cleanup') # clean utility.clean_up(utility.MSSQLCLI_DIST_DIRECTORY) utility.clean_up_egg_info_sub_directories(utility.ROOT_DIR) print_heading('Running setup') # install general requirements. utility.exec_command('{0} install -r requirements-dev.txt'.format(PIP), utility.ROOT_DIR) # convert windows line endings to unix for mssql-cli bash script utility.exec_command('{0} dos2unix.py mssql-cli mssql-cli'.format(PYTHON), utility.ROOT_DIR) # run flake8 code_analysis() if utility.get_current_platform().startswith('win'): platforms_to_build = ['win32', 'win_amd64'] else: platforms_to_build = [utility.get_current_platform()] for plat in platforms_to_build: # For the current platform, populate the appropriate binaries and # generate the wheel. clean_and_copy_sqltoolsservice(plat) utility.clean_up(utility.MSSQLCLI_BUILD_DIRECTORY) print_heading('Building mssql-cli pip package') utility.exec_command('%s --version' % PYTHON, utility.ROOT_DIR) utility.exec_command( '%s setup.py check -r -s bdist_wheel --plat-name %s' % (PYTHON, plat), utility.ROOT_DIR, continue_on_error=False) # generate sdist--only run on macOS to eliminate redundant copies when published to Azure # NOTE: macOS was arbitrarily chosen as the single OS if sys.platform == 'darwin': utility.exec_command( '%s setup.py check -r -s sdist --formats=gztar' % PYTHON, utility.ROOT_DIR, continue_on_error=False) # Copy back the SqlToolsService binaries for this platform. clean_and_copy_sqltoolsservice(utility.get_current_platform()) copy_and_rename_wheels()
def publish_official(platforms_names): """ Publish mssql-scripter wheel package to PyPi. """ mssqlscripter_wheel_dir = os.listdir(utility.MSSQLSCRIPTER_DIST_DIRECTORY) # Run twine action for mssqlscripter. # Only authorized users with credentials will be able to upload this package. # Credentials will be stored in a .pypirc file. for mssqlscripter_wheel_name in mssqlscripter_wheel_dir: utility.exec_command( 'twine upload {}'.format(mssqlscripter_wheel_name), utility.MSSQLSCRIPTER_DIST_DIRECTORY)
def verify_local_install(): """ Install mssql-scripter package locally that resolves mssqltoolsservice dependency from local build. """ # Local install of mssql-scripter. mssqlscripter_sdist_name = os.listdir( utility.MSSQLSCRIPTER_DIST_DIRECTORY)[0] # To ensure we have a clean install, we disable the cache as to prevent cache overshadowing actual changes made. utility.exec_command( 'pip install --no-cache-dir --no-index --find-links=./mssqltoolsservice/dist ./dist/{}' .format(mssqlscripter_sdist_name), root_dir, continue_on_error=False)
def tex_to_pdf(self, tex_command, latex_text, preamble_file): """ Create a PDF file from latex text """ with logger.debug("Converting .tex to .pdf"): # Read preamble preamble_file = os.path.abspath(preamble_file) preamble = "" if os.path.isfile(preamble_file): with open(preamble_file, 'r') as f: preamble += f.read() # Options pass to LaTeX-related commands texwrapper = self.DOCUMENT_TEMPLATE % (preamble, latex_text) # Convert TeX to PDF # Write tex with open(self.tmp('tex'), 'w') as f_tex: f_tex.write(texwrapper) # Exec tex_command: tex -> pdf try: exec_command([tex_command, self.tmp('tex')] + self.LATEX_OPTIONS) except TexTextCommandFailed as error: if os.path.exists(self.tmp('log')): parsed_log = self.parse_pdf_log(self.tmp('log')) raise TexTextConversionError(parsed_log, error.return_code, error.stdout, error.stderr) else: raise TexTextConversionError(str(error), error.return_code, error.stdout, error.stderr) if not os.path.exists(self.tmp('pdf')): raise TexTextConversionError( "%s didn't produce output %s" % (tex_command, self.tmp('pdf')))
def validate_package(): """ Install mssql-cli package locally. """ root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), '..')) # Local install of mssql-scripter. mssqlcli_wheel_dir = os.listdir(utility.MSSQLCLI_DIST_DIRECTORY) # To ensure we have a clean install, we disable the cache as to prevent # cache overshadowing actual changes made. current_platform = utility.get_current_platform() mssqlcli_wheel_name = [ pkge for pkge in mssqlcli_wheel_dir if current_platform in pkge ] utility.exec_command( 'pip install --no-cache-dir --no-index ./dist/{}'.format( mssqlcli_wheel_name[0]), root_dir, continue_on_error=False)
def unit_test(): """ Run all unit tests. """ utility.exec_command( 'pytest --cov mssqlcli --doctest-modules --junitxml=junit/test-results.xml --cov-report=xml --cov-report=html ' 'tests/test_mssqlcliclient.py ' 'tests/test_completion_refresher.py ' 'tests/test_config.py ' 'tests/test_naive_completion.py ' 'tests/test_main.py ' 'tests/test_fuzzy_completion.py ' 'tests/test_rowlimit.py ' 'tests/test_sqlcompletion.py ' 'tests/test_prioritization.py ' 'mssqlcli/jsonrpc/tests ' 'mssqlcli/jsonrpc/contracts/tests ' 'tests/test_telemetry.py ' 'tests/test_special.py', utility.ROOT_DIR, continue_on_error=False)
def build(): """ Builds mssql-cli package. """ print_heading('Cleanup') # clean utility.clean_up(utility.MSSQLCLI_DIST_DIRECTORY) utility.clean_up_egg_info_sub_directories(utility.ROOT_DIR) print_heading('Running setup') # install general requirements. utility.exec_command('{0} install -r requirements-dev.txt'.format(PIP), utility.ROOT_DIR) # convert windows line endings to unix for mssql-cli bash script utility.exec_command('{0} dos2unix.py mssql-cli mssql-cli'.format(PYTHON), utility.ROOT_DIR) # run flake8 code_analysis() if utility.get_current_platform().startswith('win'): platforms_to_build = ['win32', 'win_amd64'] else: platforms_to_build = [utility.get_current_platform()] for platform in platforms_to_build: # For the current platform, populate the appropriate binaries and # generate the wheel. clean_and_copy_sqltoolsservice(platform) utility.clean_up(utility.MSSQLCLI_BUILD_DIRECTORY) print_heading('Building mssql-cli pip package') utility.exec_command('{0} --version'.format(PYTHON), utility.ROOT_DIR) utility.exec_command( '{0} setup.py check -r -s bdist_wheel --plat-name {1}'.format( PYTHON, platform), utility.ROOT_DIR, continue_on_error=False) # Copy back the SqlToolsService binaries for this platform. clean_and_copy_sqltoolsservice(utility.get_current_platform()) copy_and_rename_wheels()
def build(platform_names): """ Builds mssql-scripter package. """ print_heading('Cleanup') # clean utility.clean_up(utility.MSSQLSCRIPTER_DIST_DIRECTORY) print_heading('Running setup') # install general requirements. utility.exec_command('pip install -r dev_requirements.txt', utility.ROOT_DIR) # convert windows line endings to unix for mssql-cli bash script utility.exec_command('python dos2unix.py mssql-scripter mssql-scripter', utility.ROOT_DIR) for platform in platform_names: utility.clean_up(utility.MSSQLSCRIPTER_BUILD_DIRECTORY) utility.cleaun_up_egg_info_sub_directories(utility.ROOT_DIR) mssqltoolsservice.copy_sqltoolsservice(platform) print_heading( 'Building mssql-scripter {} wheel package package'.format( platform)) utility.exec_command('python --version', utility.ROOT_DIR) utility.exec_command( 'python setup.py check -r -s bdist_wheel --plat-name {}'.format( platform), utility.ROOT_DIR, continue_on_error=False) mssqltoolsservice.clean_up_sqltoolsservice()
def code_analysis(): utility.exec_command( '{0} -m flake8 mssqlcli setup.py dev_setup.py build.py utility.py dos2unix.py'.format(PYTHON), utility.ROOT_DIR)
def code_analysis(): utility.exec_command( '%s -m flake8 mssqlcli setup.py dev_setup.py build.py utility.py dos2unix.py' % PYTHON, utility.ROOT_DIR)
#!/usr/bin/env python from __future__ import print_function import os import sys import utility PYTHON = os.getenv('CUSTOM_PYTHON', sys.executable) print('Running dev setup...') print('Root directory \'%s\'\n' % utility.ROOT_DIR) # install general requirements. utility.exec_command('%s -m pip install --no-cache-dir -r requirements-dev.txt' % PYTHON, utility.ROOT_DIR) import mssqlcli.mssqltoolsservice.externals as mssqltoolsservice # download the sqltoolssevice binaries for all platforms mssqltoolsservice.download_sqltoolsservice_binaries() # install mssqltoolsservice if this platform supports it. utility.copy_current_platform_mssqltoolsservice() print('Finished dev setup.')
#!/usr/bin/env python # -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- from __future__ import print_function import os import platform import utility import mssqlscripter.mssqltoolsservice.external as mssqltoolsservice print('Running dev setup...') print('Root directory \'{}\'\n'.format(utility.ROOT_DIR)) # install general requirements. utility.exec_command('pip install -r dev_requirements.txt', utility.ROOT_DIR) run_time_id = utility.get_current_platform() if run_time_id: mssqltoolsservice.copy_sqltoolsservice(run_time_id) else: print("This platform does not support mssqltoolsservice.") print('Finished dev setup.')
def build(options): supported_actions = ['nightly'] action = None if len(options) >= 1: if options[0] not in supported_actions: print('Please provide a supported action {}.'.format( supported_actions)) return action = options[0] if action == 'nightly': assert AZURE_STORAGE_CONNECTION_STRING, 'Set AZURE_STORAGE_CONNECTION_STRING environment variable' print_heading('Cleanup') # clean utility.clean_up(utility.MSSQLSCRIPTER_DIST_DIRECTORY) utility.clean_up(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY) utility.cleaun_up_egg_info_sub_directories(utility.ROOT_DIR) utility.cleaun_up_egg_info_sub_directories( utility.MSSQLTOOLSSERVICE_DIRECTORY) print_heading('Running setup') # install general requirements. utility.exec_command('pip install -r dev_requirements.txt', utility.ROOT_DIR) print_heading('Running mssql-scripter tests') utility.exec_command('tox', utility.ROOT_DIR, continue_on_error=False) print_heading('Building mssql-scripter pip package') utility.exec_command('python setup.py check -r -s sdist', utility.ROOT_DIR, continue_on_error=False) print_heading('Building mssqltoolsservice pip package') utility.exec_command('python buildwheels.py', utility.MSSQLTOOLSSERVICE_DIRECTORY, continue_on_error=False) if action == 'nightly': blob_service = BlockBlobService( connection_string=AZURE_STORAGE_CONNECTION_STRING) print_heading('Uploading packages to blob storage ') for pkg in os.listdir(utility.MSSQLSCRIPTER_DIST_DIRECTORY): pkg_path = os.path.join(utility.MSSQLSCRIPTER_DIST_DIRECTORY, pkg) print('Uploading package {}'.format(pkg_path)) upload_package(blob_service, pkg_path, 'mssql-scripter') for pkg in os.listdir(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY): pkg_path = os.path.join(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY, pkg) pkg_name = os.path.basename(pkg_path).split('-')[0].replace( '_', '-').lower() print('Uploading package {}'.format(pkg_name)) upload_package(blob_service, pkg_path, pkg_name) # Upload the final index file upload_index_file(blob_service, 'index.html', 'Simple Index', UPLOADED_PACKAGE_LINKS)
#!/usr/bin/env python from __future__ import print_function import os import utility PIP = os.getenv('CUSTOM_PIP', 'pip') print('Running dev setup...') print('Root directory \'%s\'\n' % utility.ROOT_DIR) # install general requirements. utility.exec_command('%s install --no-cache-dir -r requirements-dev.txt' % PIP, utility.ROOT_DIR) # install mssqltoolsservice if this platform supports it. utility.copy_current_platform_mssqltoolsservice() print('Finished dev setup.')
#!/usr/bin/env python # -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- from __future__ import print_function import os import setup import utility root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), '..')) print('Running dev setup...') print('Root directory \'{}\'\n'.format(root_dir)) # install general requirements. utility.exec_command('pip install -r dev_requirements.txt', root_dir) # install mssqltoolsservice if this platform supports it. mssqltoolsservice_package_name = os.environ['MSSQLTOOLSSERVICE_PACKAGE_NAME'] print('Installing {}...'.format(mssqltoolsservice_package_name)) # mssqltoolsservice package name is retrieved from environment variable set by setup.py. utility.exec_command('pip install {}'.format(mssqltoolsservice_package_name), root_dir) print('Finished dev setup.')
#!/usr/bin/env python from __future__ import print_function import os import utility PIP = os.getenv('CUSTOM_PIP', 'pip') print('Running dev setup...') print('Root directory \'{}\'\n'.format(utility.ROOT_DIR)) # install general requirements. utility.exec_command( '{0} install --no-cache-dir -r requirements-dev.txt'.format(PIP), utility.ROOT_DIR) # install mssqltoolsservice if this platform supports it. utility.copy_current_platform_mssqltoolsservice() print('Finished dev setup.')