def upload(path,
imagestore_string='fabric:ImageStore',
show_progress=False,
timeout=300): # pylint: disable=too-many-locals,missing-docstring
from sfctl.config import (client_endpoint, no_verify_setting, ca_cert_info,
cert_info)
import requests
abspath = validate_app_path(path)
basename = os.path.basename(abspath)
endpoint = client_endpoint()
cert = cert_info()
ca_cert = True
if no_verify_setting():
ca_cert = False
elif ca_cert_info():
ca_cert = ca_cert_info()
if all([no_verify_setting(), ca_cert_info()]):
raise CLIError('Cannot specify both CA cert info and no verify')
# Note: pressing ctrl + C during upload does not end the current upload in progress, but only
# stops the next one from occurring. This will be fixed in the future.
# Upload to either to a folder, or native image store only
if 'file:' in imagestore_string:
dest_path = path_from_imagestore_string(imagestore_string)
process = Process(target=upload_to_fileshare,
args=(abspath, os.path.join(dest_path, basename),
show_progress))
process.start()
process.join(timeout) # If timeout is None then there is no timeout.
if process.is_alive():
process.terminate(
) # This will leave any children of process orphaned.
raise SFCTLInternalException(
'Upload has timed out. Consider passing a longer '
'timeout duration.')
elif imagestore_string == 'fabric:ImageStore':
with requests.Session() as sesh:
sesh.verify = ca_cert
sesh.cert = cert
# There is no need for a new process here since
upload_to_native_imagestore(sesh, endpoint, abspath, basename,
show_progress, timeout)
else:
raise CLIError('Unsupported image store connection string')
def upload_single_file_native_imagestore(sesh, endpoint, basename, #pylint: disable=too-many-locals,too-many-arguments
rel_path, single_file, root, target_timeout):
"""
Used by upload_to_native_imagestore to upload individual files
of the application package to cluster
:param sesh: A requests (module) session object.
:param endpoint: Connection url endpoint for upload requests.
:param basename: Image store base path.
:param rel_path: Image store relative directory path.
:param single_file: Filename.
:param root: Source directory path.
:param target_timeout: Time at which timeout would be reached.
"""
try:
from urllib.parse import urlparse, urlencode, urlunparse
except ImportError:
from urllib import urlencode
from urlparse import urlparse, urlunparse # pylint: disable=import-error
current_time_left = get_timeout_left(target_timeout) # an int representing seconds
if current_time_left == 0:
raise SFCTLInternalException('Upload has timed out. Consider passing a longer '
'timeout duration.')
url_path = (
os.path.normpath(os.path.join('ImageStore', basename,
rel_path, single_file))
).replace('\\', '/')
fp_norm = os.path.normpath(os.path.join(root, single_file))
with open(fp_norm, 'rb') as file_opened:
url_parsed = list(urlparse(endpoint))
url_parsed[2] = url_path
url_parsed[4] = urlencode(
{'api-version': '6.1',
'timeout': current_time_left})
url = urlunparse(url_parsed)
# timeout is (connect_timeout, read_timeout)
res = sesh.put(url, data=file_opened,
timeout=(get_lesser(60, current_time_left), current_time_left))
res.raise_for_status()
def sfctl_cluster_version_matches(cluster_version, sfctl_version):
"""
Check if the sfctl version and the cluster version is compatible with each other.
:param cluster_version: str representing the cluster runtime version of the connected cluster
:param sfctl_version: str representing this sfctl distribution version
:return: True if they are a match. False otherwise.
"""
if sfctl_version in ['11.0.0']:
return cluster_version.startswith('7.2')
# If we forget to update this code before a new release, the tests which call this method
# will fail.
raise SFCTLInternalException(
str.format(
'Invalid sfctl version {0} provided for check against cluster version {1}.',
sfctl_version, cluster_version))
def upload(
path,
imagestore_string='fabric:ImageStore',
show_progress=False,
timeout=300, # pylint: disable=too-many-locals,missing-docstring,too-many-arguments,too-many-branches,too-many-statements
compress=False,
keep_compressed=False,
compressed_location=None):
from sfctl.config import (client_endpoint, no_verify_setting, ca_cert_info,
cert_info)
import requests
path = _normalize_path(path)
if compressed_location is not None:
compressed_location = _normalize_path(compressed_location)
abspath = validate_app_path(path)
basename = os.path.basename(abspath)
endpoint = client_endpoint()
cert = cert_info()
ca_cert = True
if no_verify_setting():
ca_cert = False
elif ca_cert_info():
ca_cert = ca_cert_info()
if all([no_verify_setting(), ca_cert_info()]):
raise CLIError('Cannot specify both CA cert info and no verify')
if not compress and (keep_compressed or compressed_location is not None):
raise CLIError(
'--keep-compressed and --compressed-location options are only applicable '
'if the --compress option is set')
compressed_pkg_location = None
created_dir_path = None
if compress:
parent_folder = os.path.dirname(path)
file_or_folder_name = os.path.basename(path)
compressed_pkg_location = os.path.join(parent_folder,
'sfctl_compressed_temp')
if compressed_location is not None:
compressed_pkg_location = compressed_location
# Check if a zip file has already been created
created_dir_path = os.path.join(compressed_pkg_location,
file_or_folder_name)
if os.path.exists(created_dir_path):
if get_user_confirmation(
str.format(
'Deleting previously generated compressed files at '
'{0}. If this folder has anything else, those will be '
'deleted as well. Allow? ["y", "n"]: ',
created_dir_path)):
shutil.rmtree(created_dir_path)
else:
# We can consider adding an option to number the packages in the future.
print(
'Stopping upload operation. Cannot compress to the following location '
'because the path already exists: ' + created_dir_path)
return
# Let users know where to find the compressed app package before starting the
# copy / compression, in case the process crashes in the middle, so users
# will know where to clean up items from, or where to upload already compressed
# app packages from
if show_progress:
print('Starting package compression into location: ' +
compressed_pkg_location)
print() # New line for formatting purposes
compress_package(path, compressed_pkg_location)
# Change the path to the path with the compressed package
compressed_path = os.path.join(compressed_pkg_location,
file_or_folder_name)
# re-do validation and reset the variables
abspath = validate_app_path(compressed_path)
basename = os.path.basename(abspath)
# Note: pressing ctrl + C during upload does not end the current upload in progress, but only
# stops the next one from occurring. This will be fixed in the future.
# Upload to either to a folder, or native image store only
if 'file:' in imagestore_string:
dest_path = path_from_imagestore_string(imagestore_string)
process = Process(target=upload_to_fileshare,
args=(abspath, os.path.join(dest_path, basename),
show_progress))
process.start()
process.join(timeout) # If timeout is None then there is no timeout.
if process.is_alive():
process.terminate(
) # This will leave any children of process orphaned.
raise SFCTLInternalException(
'Upload has timed out. Consider passing a longer '
'timeout duration.')
elif imagestore_string == 'fabric:ImageStore':
with requests.Session() as sesh:
sesh.verify = ca_cert
sesh.cert = cert
# There is no need for a new process here since
upload_to_native_imagestore(sesh, endpoint, abspath, basename,
show_progress, timeout)
else:
raise CLIError(
'Unsupported image store connection string. Value should be either '
'"fabric:ImageStore", or start with "file:"')
# If code has reached here, it means that upload was successful
# To reach here, user must have agreed to clear this folder or exist the API
# So we can safely delete the contents
# User is expected to not create a folder by the same name during the upload duration
# If needed, we can consider adding our content under a GUID in the future
if compress and not keep_compressed:
# Remove the generated files
if show_progress:
print('Removing generated folder ' + created_dir_path)
shutil.rmtree(created_dir_path)
def upload_to_native_imagestore(
sesh,
endpoint,
abspath,
basename, #pylint: disable=too-many-locals,too-many-arguments
show_progress,
timeout):
"""
Upload the application package to cluster
:param sesh: A requests (module) session object.
"""
try:
from urllib.parse import urlparse, urlencode, urlunparse
except ImportError:
from urllib import urlencode
from urlparse import urlparse, urlunparse # pylint: disable=import-error
total_files_count = 0
current_files_count = 0
for root, _, files in os.walk(abspath):
# Number of uploads is number of files plus number of directories
total_files_count += (len(files) + 1)
target_timeout = int(time()) + timeout
# Note: while we are raising some exceptions regarding upload timeout, we are leaving the
# timeouts raised by the requests library as is since it contains enough information
for root, _, files in os.walk(abspath):
rel_path = os.path.normpath(os.path.relpath(root, abspath))
for single_file in files:
current_time_left = get_timeout_left(
target_timeout) # an int representing seconds
if current_time_left == 0:
raise SFCTLInternalException(
'Upload has timed out. Consider passing a longer '
'timeout duration.')
url_path = (os.path.normpath(
os.path.join('ImageStore', basename, rel_path,
single_file))).replace('\\', '/')
fp_norm = os.path.normpath(os.path.join(root, single_file))
with open(fp_norm, 'rb') as file_opened:
url_parsed = list(urlparse(endpoint))
url_parsed[2] = url_path
url_parsed[4] = urlencode({
'api-version': '6.1',
'timeout': current_time_left
})
url = urlunparse(url_parsed)
# timeout is (connect_timeout, read_timeout)
res = sesh.put(url,
data=file_opened,
timeout=(get_lesser(60, current_time_left),
current_time_left))
res.raise_for_status()
current_files_count += 1
print_progress(
current_files_count, total_files_count,
os.path.normpath(os.path.join(rel_path, single_file)),
show_progress, get_timeout_left(target_timeout))
current_time_left = get_timeout_left(target_timeout)
if current_time_left == 0:
raise SFCTLInternalException(
'Upload has timed out. Consider passing a longer '
'timeout duration.')
url_path = (os.path.normpath(
os.path.join('ImageStore', basename, rel_path,
'_.dir'))).replace('\\', '/')
url_parsed = list(urlparse(endpoint))
url_parsed[2] = url_path
url_parsed[4] = urlencode({
'api-version': '6.1',
'timeout': current_time_left
})
url = urlunparse(url_parsed)
res = sesh.put(url,
timeout=(get_lesser(60, current_time_left),
current_time_left))
res.raise_for_status()
current_files_count += 1
print_progress(current_files_count, total_files_count,
os.path.normpath(os.path.join(rel_path, '_.dir')),
show_progress, get_timeout_left(target_timeout))
if show_progress:
print('Complete', file=sys.stderr)
def provision_application_type(
client, #pylint: disable=too-many-locals,invalid-name,too-many-arguments
external_provision=False,
no_wait=False,
application_type_build_path=None,
application_package_download_uri=None,
application_type_name=None,
application_type_version=None,
timeout=60):
"""Provisions or registers a Service Fabric application type with the
cluster using the .sfpkg package in the external store or using the
application package in the image store.
"""
from azure.servicefabric.models import (
ProvisionApplicationTypeDescription,
ExternalStoreProvisionApplicationTypeDescription, FabricErrorException)
provision_description = None
# Validate inputs
if external_provision:
if application_type_build_path:
raise CLIError(
'application-type-build-path should not be specified for external provision.'
)
if not all([
application_package_download_uri, application_type_name,
application_type_version
]):
raise CLIError(
'Missing required parameters. The following are required: '
'--application-package-download-uri, --application-type-name, '
'--application-type-version.')
provision_description = ExternalStoreProvisionApplicationTypeDescription(
async_property=no_wait,
application_package_download_uri=application_package_download_uri,
application_type_name=application_type_name,
application_type_version=application_type_version)
else:
if not application_type_build_path:
raise CLIError('Missing required parameter '
'--application-type-build-path.')
if any([
application_package_download_uri, application_type_name,
application_type_version
]):
raise CLIError(
'The following are should not be specified for image store provision: '
'--application-package-download-uri, --application-type-name, '
'--application-type-version.')
provision_description = ProvisionApplicationTypeDescription(
async_property=no_wait,
application_type_build_path=application_type_build_path)
api_version = "6.2"
# Construct URLs
url = '/ApplicationTypes/$/Provision'
# Construct parameters
query_parameters = {}
query_parameters['api-version'] = client._serialize.query(
"api_version", api_version, 'str')
query_parameters['timeout'] = client._serialize.query("timeout",
timeout,
'long',
maximum=4294967295,
minimum=1)
# Construct headers
header_parameters = {}
header_parameters['Content-Type'] = 'application/json; charset=utf-8'
# Construct body
body_content = None
if not external_provision:
body_content = client._serialize.body(
provision_description, 'ProvisionApplicationTypeDescription')
else:
body_content = client._serialize.body(
provision_description,
'ExternalStoreProvisionApplicationTypeDescription')
# Create a new sorted dictionary since we don't have move_to_end in python 2
body_content_sorted = OrderedDict([('Kind', body_content['Kind'])])
for key in body_content:
if key != 'Kind':
body_content_sorted[key] = body_content[key]
if list(body_content_sorted.keys())[0] != "Kind":
raise SFCTLInternalException(
'provision_application_type: Kind must be the first item to be serialized.'
)
# Construct and send request
request = client._client.post(url, query_parameters)
response = client._client.send(request, header_parameters,
body_content_sorted)
if response.status_code not in [200, 202]:
raise FabricErrorException(client._deserialize, response)
def upload_to_native_imagestore(sesh, endpoint, abspath, basename, #pylint: disable=too-many-locals,too-many-arguments
show_progress, timeout):
"""
Upload the application package to cluster
:param sesh: A requests (module) session object.
:param endpoint: Connection url endpoint for upload requests.
:param abspath: Application source path.
:param basename: Image store destination path.
:param show_progress: boolean to determine whether to log upload progress.
:param timeout: Total upload timeout in seconds.
"""
try:
from urllib.parse import urlparse, urlencode, urlunparse
except ImportError:
from urllib import urlencode
from urlparse import urlparse, urlunparse # pylint: disable=import-error
total_files_count = 0
current_files_count = 0
for root, _, files in os.walk(abspath):
# Number of uploads is number of files plus number of directories
total_files_count += (len(files) + 1)
target_timeout = int(time()) + timeout
jobcount = get_job_count()
# Note: while we are raising some exceptions regarding upload timeout, we are leaving the
# timeouts raised by the requests library as is since it contains enough information
for root, _, files in os.walk(abspath):
rel_path = os.path.normpath(os.path.relpath(root, abspath))
filecount = len(files)
if show_progress:
progressdescription = 'Uploading path: {}'.format(rel_path)
with tqdm_joblib(tqdm(desc=progressdescription, total=filecount)):
Parallel(n_jobs=jobcount)(
delayed(upload_single_file_native_imagestore)(
sesh, endpoint, basename, rel_path, single_file, root, target_timeout)
for single_file in files)
else:
Parallel(n_jobs=jobcount)(
delayed(upload_single_file_native_imagestore)(
sesh, endpoint, basename, rel_path, single_file, root, target_timeout)
for single_file in files)
current_time_left = get_timeout_left(target_timeout)
if current_time_left == 0:
raise SFCTLInternalException('Upload has timed out. Consider passing a longer '
'timeout duration.')
url_path = (
os.path.normpath(os.path.join('ImageStore', basename,
rel_path, '_.dir'))
).replace('\\', '/')
url_parsed = list(urlparse(endpoint))
url_parsed[2] = url_path
url_parsed[4] = urlencode({'api-version': '6.1',
'timeout': current_time_left})
url = urlunparse(url_parsed)
res = sesh.put(url,
timeout=(get_lesser(60, current_time_left), current_time_left))
res.raise_for_status()
current_files_count += filecount + 1
print_progress(current_files_count, total_files_count,
os.path.normpath(os.path.join(rel_path, '_.dir')),
show_progress, get_timeout_left(target_timeout))
if show_progress:
print('Complete', file=sys.stderr)