def _init_task(self): super(SchedulePushOrgList, self)._init_task() self.push = SalesforcePushApi( self.sf, self.logger, self.options['batch_size'], )
def _get_orgs(self): subscriber_where = self.options.get('subscriber_where') default_where = {'PackageSubscriber': "OrgStatus = 'Active' AND InstalledStatus = 'i'"} if subscriber_where: default_where['PackageSubscriber'] += " AND ({})".format(subscriber_where) push_api = SalesforcePushApi(self.sf, self.logger, default_where=default_where.copy()) package = self._get_package(self.options.get('namespace')) version = self._get_version(package, self.options.get('version')) min_version = self.options.get('min_version') if min_version: min_version = self._get_version(package, self.options.get('min_version')) orgs = [] if min_version: # If working with a range of versions, use an inclusive search versions = version.get_older_released_version_objs(greater_than_version=min_version) included_versions = [] for include_version in versions: included_versions.append(str(include_version.sf_id)) if not included_versions: raise ValueError( 'No versions found between version id {} and {}'.format( version.version_number, min_version.version_number ) ) # Query orgs for each version in the range individually to avoid query timeout errors with querying multiple versions for included_version in included_versions: # Clear the get_subscribers method cache before each call push_api.get_subscribers.cache.clear() push_api.default_where['PackageSubscriber'] = "{} AND MetadataPackageVersionId = '{}'".format( default_where['PackageSubscriber'], included_version, ) for subscriber in push_api.get_subscribers(): orgs.append(subscriber['OrgKey']) else: # If working with a specific version rather than a range, use an exclusive search # Add exclusion of all orgs running on newer releases newer_versions = version.get_newer_released_version_objs() excluded_versions = [str(version.sf_id),] for newer in newer_versions: excluded_versions.append(str(newer.sf_id)) if len(excluded_versions) == 1: push_api.default_where['PackageSubscriber'] += " AND MetadataPackageVersionId != '{}'".format( excluded_versions[0], ) else: push_api.default_where['PackageSubscriber'] += " AND MetadataPackageVersionId NOT IN {}".format( "('" + "','".join(excluded_versions) + "')" ) for subscriber in push_api.get_subscribers(): orgs.append(subscriber['OrgKey']) return orgs
def _get_push_request_query(self, request_id): default_where = {"PackagePushRequest": "Id = '{}'".format(request_id)} # Create a new PushAPI instance with different settings than self.push self.push_report = SalesforcePushApi( self.sf, self.logger, lazy=["subscribers", "jobs"], default_where=default_where, ) # Get the push request self.push_request = self.push_report.get_push_request_objs( "Id = '{}'".format(request_id), limit=1 ) if not self.push_request: raise PushApiObjectNotFound( "Push Request {} was not found".format(self.push_request) ) self.push_request = self.push_request[0]
class BaseSalesforcePushTask(BaseSalesforceApiTask): completed_statuses = ['Succeeded', 'Failed', 'Canceled'] api_version = '38.0' def _init_task(self): super(BaseSalesforcePushTask, self)._init_task() self.push = SalesforcePushApi(self.sf, self.logger) def _parse_version(self, version): # Parse the version number string major = None minor = None patch = None build = None state = 'Released' version_parts = version.split('.') if len(version_parts) >= 1: major = version_parts[0] if len(version_parts) == 2: minor = version_parts[1] if minor.find('Beta') != -1: state = 'Beta' minor, build = minor.replace( ' (Beta ', ',', ).replace(')', '').split(',') if len(version_parts) > 2: minor = version_parts[1] patch = version_parts[2] if patch.find('Beta') != -1: state = 'Beta' patch, build = minor.replace( ' (Beta ', ',', ).replace(')', '').split(',') return { 'major': major, 'minor': minor, 'patch': patch, 'build': build, 'state': state, } def _get_version(self, package, version): version_info = self._parse_version(version) version_where = ( "ReleaseState = '{}'".format(version_info['state']) + " AND MajorVersion = {}".format(version_info['major']) + " AND MinorVersion = {}".format(version_info['minor'])) if version_info.get('patch'): version_where += " AND PatchVersion = {}".format( version_info['patch']) if version_info['state'] == 'Beta' and version_info.get('build'): version_where += " AND BuildNumber = {}".format( version_info['build']) version = package.get_package_version_objs(version_where, limit=1) if not version: raise PushApiObjectNotFound( 'PackageVersion not found.' + ' Namespace = {}, Version Info = {}'.format( package.namespace, version_info, )) return version[0] def _get_package(self, namespace): package = self.push.get_package_objs( "NamespacePrefix = '{}'".format(namespace), limit=1) if not package: raise PushApiObjectNotFound( 'The package with namespace {} was not found'.format( namespace)) return package[0] def _load_orgs_file(self, path): orgs = [] with open(path, 'r') as f: for line in f: if line.isspace(): continue orgs.append(line.split()[0]) return orgs def _report_push_status(self, request_id): default_where = {'PackagePushRequest': "Id = '{}'".format(request_id)} # Create a new PushAPI instance with different settings than self.push self.push_report = SalesforcePushApi( self.sf, self.logger, lazy=['subscribers', 'jobs'], default_where=default_where, ) # Get the push request push_request = self.push_report.get_push_request_objs( "Id = '{}'".format(request_id), limit=1, ) if not push_request: raise PushApiObjectNotFound( 'Push Request {} was not found'.format(push_request)) push_request = push_request[0] # Check if the request is complete interval = 10 if push_request.status not in self.completed_statuses: self.logger.info( 'Push request is not yet complete.' + ' Polling for status every {} seconds until completion'.format( interval)) # Loop waiting for request completion i = 0 while push_request.status not in self.completed_statuses: if i == 10: self.logger.info( 'This is taking a while! Polling every 60 seconds') interval = 60 time.sleep(interval) # Clear the method level cache on get_push_requests and # get_push_request_objs self.push_report.get_push_requests.cache.clear() self.push_report.get_push_request_objs.cache.clear() # Get the push_request again push_request = self.push_report.get_push_request_objs( "Id = '{}'".format(request_id), limit=1, )[0] self.logger.info(push_request.status) i += 1 failed_jobs = [] success_jobs = [] canceled_jobs = [] jobs = push_request.get_push_job_objs() for job in jobs: if job.status == 'Failed': failed_jobs.append(job) elif job.status == 'Succeeded': success_jobs.append(job) elif job.status == 'Canceled': canceled_jobs.append(job) self.logger.info( "Push complete: {} succeeded, {} failed, {} canceled".format( len(success_jobs), len(failed_jobs), len(canceled_jobs), )) failed_by_error = {} for job in failed_jobs: errors = job.get_push_error_objs() for error in errors: error_key = ( error.error_type, error.title, error.message, error.details, ) if error_key not in failed_by_error: failed_by_error[error_key] = [] failed_by_error[error_key].append(error) if failed_jobs: self.logger.info("-----------------------------------") self.logger.info("Failures by error type") self.logger.info("-----------------------------------") for key, errors in failed_by_error.items(): self.logger.info(" ") self.logger.info("{} failed with...".format(len(errors))) self.logger.info(" Error Type = {}".format(key[0])) self.logger.info(" Title = {}".format(key[1])) self.logger.info(" Message = {}".format(key[2])) self.logger.info(" Details = {}".format(key[3]))
def _report_push_status(self, request_id): default_where = {'PackagePushRequest': "Id = '{}'".format(request_id)} # Create a new PushAPI instance with different settings than self.push self.push_report = SalesforcePushApi( self.sf, self.logger, lazy=['subscribers', 'jobs'], default_where=default_where, ) # Get the push request push_request = self.push_report.get_push_request_objs( "Id = '{}'".format(request_id), limit=1, ) if not push_request: raise PushApiObjectNotFound( 'Push Request {} was not found'.format(push_request)) push_request = push_request[0] # Check if the request is complete interval = 10 if push_request.status not in self.completed_statuses: self.logger.info( 'Push request is not yet complete.' + ' Polling for status every {} seconds until completion'.format( interval)) # Loop waiting for request completion i = 0 while push_request.status not in self.completed_statuses: if i == 10: self.logger.info( 'This is taking a while! Polling every 60 seconds') interval = 60 time.sleep(interval) # Clear the method level cache on get_push_requests and # get_push_request_objs self.push_report.get_push_requests.cache.clear() self.push_report.get_push_request_objs.cache.clear() # Get the push_request again push_request = self.push_report.get_push_request_objs( "Id = '{}'".format(request_id), limit=1, )[0] self.logger.info(push_request.status) i += 1 failed_jobs = [] success_jobs = [] canceled_jobs = [] jobs = push_request.get_push_job_objs() for job in jobs: if job.status == 'Failed': failed_jobs.append(job) elif job.status == 'Succeeded': success_jobs.append(job) elif job.status == 'Canceled': canceled_jobs.append(job) self.logger.info( "Push complete: {} succeeded, {} failed, {} canceled".format( len(success_jobs), len(failed_jobs), len(canceled_jobs), )) failed_by_error = {} for job in failed_jobs: errors = job.get_push_error_objs() for error in errors: error_key = ( error.error_type, error.title, error.message, error.details, ) if error_key not in failed_by_error: failed_by_error[error_key] = [] failed_by_error[error_key].append(error) if failed_jobs: self.logger.info("-----------------------------------") self.logger.info("Failures by error type") self.logger.info("-----------------------------------") for key, errors in failed_by_error.items(): self.logger.info(" ") self.logger.info("{} failed with...".format(len(errors))) self.logger.info(" Error Type = {}".format(key[0])) self.logger.info(" Title = {}".format(key[1])) self.logger.info(" Message = {}".format(key[2])) self.logger.info(" Details = {}".format(key[3]))
def _report_push_status(self, request_id): default_where = {"PackagePushRequest": "Id = '{}'".format(request_id)} # Create a new PushAPI instance with different settings than self.push self.push_report = SalesforcePushApi( self.sf, self.logger, lazy=["subscribers", "jobs"], default_where=default_where, ) # Get the push request push_request = self.push_report.get_push_request_objs( "Id = '{}'".format(request_id), limit=1 ) if not push_request: raise PushApiObjectNotFound( "Push Request {} was not found".format(push_request) ) push_request = push_request[0] # Check if the request is complete interval = 10 if push_request.status not in self.completed_statuses: self.logger.info( "Push request is not yet complete." + " Polling for status every {} seconds until completion".format( interval ) ) # Loop waiting for request completion i = 0 while push_request.status not in self.completed_statuses: if i == 10: self.logger.info("This is taking a while! Polling every 60 seconds") interval = 60 time.sleep(interval) # Clear the method level cache on get_push_requests and # get_push_request_objs self.push_report.get_push_requests.cache.clear() self.push_report.get_push_request_objs.cache.clear() # Get the push_request again push_request = self.push_report.get_push_request_objs( "Id = '{}'".format(request_id), limit=1 )[0] self.logger.info(push_request.status) i += 1 failed_jobs = [] success_jobs = [] canceled_jobs = [] jobs = push_request.get_push_job_objs() for job in jobs: if job.status == "Failed": failed_jobs.append(job) elif job.status == "Succeeded": success_jobs.append(job) elif job.status == "Canceled": canceled_jobs.append(job) self.logger.info( "Push complete: {} succeeded, {} failed, {} canceled".format( len(success_jobs), len(failed_jobs), len(canceled_jobs) ) ) failed_by_error = {} for job in failed_jobs: errors = job.get_push_error_objs() for error in errors: error_key = ( error.error_type, error.title, error.message, error.details, ) if error_key not in failed_by_error: failed_by_error[error_key] = [] failed_by_error[error_key].append(error) if failed_jobs: self.logger.info("-----------------------------------") self.logger.info("Failures by error type") self.logger.info("-----------------------------------") for key, errors in failed_by_error.items(): self.logger.info(" ") self.logger.info("{} failed with...".format(len(errors))) self.logger.info(" Error Type = {}".format(key[0])) self.logger.info(" Title = {}".format(key[1])) self.logger.info(" Message = {}".format(key[2])) self.logger.info(" Details = {}".format(key[3]))
def _init_task(self): super(SchedulePushOrgList, self)._init_task() self.push = SalesforcePushApi(self.sf, self.logger, self.options["batch_size"])
class SchedulePushOrgList(BaseSalesforcePushTask): task_options = { "orgs": { "description": "The path to a file containing one OrgID per line.", "required": True, }, "version": { "description": "The managed package version to push", "required": True, }, "namespace": { "description": ( "The managed package namespace to push." + " Defaults to project__package__namespace." ) }, "start_time": { "description": ( "Set the start time (UTC) to queue a future push." + " Ex: 2016-10-19T10:00" ) }, "batch_size": { "description": ( "Break pull requests into batches of this many orgs." + " Defaults to 200." ) }, } def _init_task(self): super(SchedulePushOrgList, self)._init_task() self.push = SalesforcePushApi(self.sf, self.logger, self.options["batch_size"]) def _init_options(self, kwargs): super(SchedulePushOrgList, self)._init_options(kwargs) # Set the namespace option to the value from cumulusci.yml if not # already set if not "namespace" in self.options: self.options["namespace"] = self.project_config.project__package__namespace if not "batch_size" in self.options: self.options["batch_size"] = 200 def _get_orgs(self): return self._load_orgs_file(self.options.get("orgs")) def _run_task(self): orgs = self._get_orgs() package = self._get_package(self.options.get("namespace")) version = self._get_version(package, self.options.get("version")) start_time = self.options.get("start_time") if start_time: if start_time.lower() == "now": start_time = datetime.utcnow() + timedelta(seconds=5) else: start_time = datetime.strptime(start_time, "%Y-%m-%dT%H:%M") if start_time < datetime.utcnow(): raise CumulusCIException("Start time cannot be in the past") else: # delay a bit to allow for review delay_minutes = 5 self.logger.warning( "Scheduling push for %d minutes from now", delay_minutes ) start_time = datetime.utcnow() + timedelta(minutes=delay_minutes) self.request_id, num_scheduled_orgs = self.push.create_push_request( version, orgs, start_time ) self.return_values["request_id"] = self.request_id if num_scheduled_orgs > 1000: sleep_time_s = 30 self.logger.info( "Delaying {} seconds to allow all jobs to initialize".format( sleep_time_s ) ) time.sleep(sleep_time_s) elif num_scheduled_orgs == 0: self.logger.warning("Canceling push request with 0 orgs") self.push.cancel_push_request return self.logger.info("Setting status to Pending to queue execution.") self.logger.info("The push upgrade will start at UTC {}".format(start_time)) # Run the job self.logger.info(self.push.run_push_request(self.request_id)) self.logger.info( "Push Request {} is queued for execution.".format(self.request_id) ) # Report the status if start time is less than 1 minute from now if start_time - datetime.utcnow() < timedelta(minutes=1): self._report_push_status(self.request_id) else: self.logger.info("Exiting early since request is in the future")
class SchedulePushOrgList(BaseSalesforcePushTask): task_options = { 'orgs': { 'description': "The path to a file containing one OrgID per line.", 'required': True, }, 'version': { 'description': "The managed package version to push", 'required': True, }, 'namespace': { 'description': "The managed package namespace to push. Defaults to project__package__namespace.", }, 'start_time': { 'description': "Set the start time (UTC) to queue a future push. Ex: 2016-10-19T10:00", }, 'batch_size': { 'description': "Set batch size used to add individual orgs to the push request. Defaults to 200", }, } def _init_task(self): super(SchedulePushOrgList, self)._init_task() self.push = SalesforcePushApi(self.sf, self.logger, self.options['batch_size']) def _init_options(self, kwargs): super(SchedulePushOrgList, self)._init_options(kwargs) # Set the namespace option to the value from cumulusci.yml if not already set if not 'namespace' in self.options: self.options[ 'namespace'] = self.project_config.project__package__namespace if not 'batch_size' in self.options: self.options['batch_size'] = 200 def _get_orgs(self): return self._load_orgs_file(self.options.get('orgs')) def _run_task(self): orgs = self._get_orgs() package = self._get_package(self.options.get('namespace')) version = self._get_version(package, self.options.get('version')) start_time = self.options.get('start_time') if start_time: start_time = datetime.strptime( start_time, "%Y-%m-%dT%H:%M") # Example: 2016-10-19T10:00 self.request_id = self.push.create_push_request( version, orgs, start_time) if len(orgs) > 1000: self.logger.info( "Delaying 30 seconds to allow all jobs to initialize...") time.sleep(30) self.logger.info('Push Request {} is populated with {} orgs'.format( self.request_id, len(orgs))) self.logger.info('Setting status to Pending to queue execution.') if start_time: self.logger.info( 'The push request will start at {}'.format(start_time)) # Run the job self.logger.info(self.push.run_push_request(self.request_id)) self.logger.info('Push Request {} is queued for execution.'.format( self.request_id)) # Report the status self._report_push_status(self.request_id)
class BaseSalesforcePushTask(BaseSalesforceApiTask): completed_statuses = ["Succeeded", "Failed", "Canceled"] api_version = "38.0" def _init_task(self): super(BaseSalesforcePushTask, self)._init_task() self.push = SalesforcePushApi(self.sf, self.logger) def _parse_version(self, version): # Parse the version number string major = None minor = None patch = None build = None state = "Released" version_parts = version.split(".") if len(version_parts) >= 1: major = version_parts[0] if len(version_parts) == 2: minor = version_parts[1] if minor.find("Beta") != -1: state = "Beta" minor, build = minor.replace(" (Beta ", ",").replace(")", "").split(",") if len(version_parts) > 2: minor = version_parts[1] patch = version_parts[2] if patch.find("Beta") != -1: state = "Beta" patch, build = minor.replace(" (Beta ", ",").replace(")", "").split(",") return { "major": major, "minor": minor, "patch": patch, "build": build, "state": state, } def _get_version(self, package, version): version_info = self._parse_version(version) version_where = ( "ReleaseState = '{}'".format(version_info["state"]) + " AND MajorVersion = {}".format(version_info["major"]) + " AND MinorVersion = {}".format(version_info["minor"]) ) if version_info.get("patch"): version_where += " AND PatchVersion = {}".format(version_info["patch"]) if version_info["state"] == "Beta" and version_info.get("build"): version_where += " AND BuildNumber = {}".format(version_info["build"]) version = package.get_package_version_objs(version_where, limit=1) if not version: raise PushApiObjectNotFound( "PackageVersion not found." + " Namespace = {}, Version Info = {}".format( package.namespace, version_info ) ) return version[0] def _get_package(self, namespace): package = self.push.get_package_objs( "NamespacePrefix = '{}'".format(namespace), limit=1 ) if not package: raise PushApiObjectNotFound( "The package with namespace {} was not found".format(namespace) ) return package[0] def _load_orgs_file(self, path): orgs = [] with open(path, "r") as f: for line in f: if line.isspace(): continue orgs.append(line.split()[0]) return orgs def _get_push_request_query(self, request_id): default_where = {"PackagePushRequest": "Id = '{}'".format(request_id)} # Create a new PushAPI instance with different settings than self.push self.push_report = SalesforcePushApi( self.sf, self.logger, lazy=["subscribers", "jobs"], default_where=default_where, ) # Get the push request self.push_request = self.push_report.get_push_request_objs( "Id = '{}'".format(request_id), limit=1 ) if not self.push_request: raise PushApiObjectNotFound( "Push Request {} was not found".format(self.push_request) ) self.push_request = self.push_request[0] def _get_push_request_job_results(self): failed_jobs = [] success_jobs = [] canceled_jobs = [] jobs = self.push_request.get_push_job_objs() for job in jobs: if job.status == "Failed": failed_jobs.append(job) elif job.status == "Succeeded": success_jobs.append(job) elif job.status == "Canceled": canceled_jobs.append(job) self.logger.info( "Push complete: {} succeeded, {} failed, {} canceled".format( len(success_jobs), len(failed_jobs), len(canceled_jobs) ) ) failed_by_error = {} for job in failed_jobs: errors = job.get_push_error_objs() for error in errors: error_key = ( error.error_type, error.title, error.message, error.details, ) if error_key not in failed_by_error: failed_by_error[error_key] = [] failed_by_error[error_key].append(error) if failed_jobs: self.logger.info("-----------------------------------") self.logger.info("Failures by error type") self.logger.info("-----------------------------------") for key, errors in failed_by_error.items(): self.logger.info(" ") self.logger.info("{} failed with...".format(len(errors))) self.logger.info(" Error Type = {}".format(key[0])) self.logger.info(" Title = {}".format(key[1])) self.logger.info(" Message = {}".format(key[2])) self.logger.info(" Details = {}".format(key[3])) def _report_push_status(self, request_id): self._get_push_request_query(request_id) # Check if the request is complete interval = 10 if self.push_request.status not in self.completed_statuses: self.logger.info( "Push request is not yet complete." + " Polling for status every {} seconds until completion".format( interval ) ) # Loop waiting for request completion i = 0 while self.push_request.status not in self.completed_statuses: if i == 10: self.logger.info("This is taking a while! Polling every 60 seconds") interval = 60 time.sleep(interval) # Clear the method level cache on get_push_requests and # get_push_request_objs self.push_report.get_push_requests.cache.clear() self.push_report.get_push_request_objs.cache.clear() # Get the push_request again self.push_request = self.push_report.get_push_request_objs( "Id = '{}'".format(request_id), limit=1 )[0] self.logger.info(self.push_request.status) i += 1 self._get_push_request_job_results()
class SchedulePushOrgList(BaseSalesforcePushTask): task_options = { "csv": {"description": "The path to a CSV file to read.", "required": False}, "csv_field_name": { "description": "The CSV field name that contains organization IDs. Defaults to 'OrganizationID'", "required": False, }, "orgs": { "description": "The path to a file containing one OrgID per line.", "required": False, }, "version": { "description": "The managed package version to push", "required": True, }, "namespace": { "description": ( "The managed package namespace to push." + " Defaults to project__package__namespace." ) }, "start_time": { "description": ( "Set the start time (UTC) to queue a future push." + " Ex: 2016-10-19T10:00" ) }, "batch_size": { "description": ( "Break pull requests into batches of this many orgs." + " Defaults to 200." ) }, } def _init_task(self): super(SchedulePushOrgList, self)._init_task() self.push = SalesforcePushApi(self.sf, self.logger, self.options["batch_size"]) def _init_options(self, kwargs): super(SchedulePushOrgList, self)._init_options(kwargs) neither_file_option = "orgs" not in self.options and "csv" not in self.options both_file_options = "orgs" in self.options and "csv" in self.options if neither_file_option or both_file_options: raise TaskOptionsError( "Please call this task with either the `orgs` or `csv` option." ) # Set the namespace option to the value from cumulusci.yml if not # already set if "namespace" not in self.options: self.options["namespace"] = self.project_config.project__package__namespace if "batch_size" not in self.options: self.options["batch_size"] = 200 if "csv" not in self.options and "csv_field_name" in self.options: raise TaskOptionsError("Please provide a csv file for this task to run.") def _get_orgs(self): if "csv" in self.options: with open(self.options.get("csv"), newline="", encoding="utf-8") as csvfile: reader = csv.DictReader(csvfile) return [ row[self.options.get("csv_field_name", "OrganizationId")] for row in reader ] else: return self._load_orgs_file(self.options.get("orgs")) def _run_task(self): orgs = self._get_orgs() package = self._get_package(self.options.get("namespace")) version = self._get_version(package, self.options.get("version")) start_time = self.options.get("start_time") if start_time: if start_time.lower() == "now": start_time = datetime.utcnow() + timedelta(seconds=5) else: start_time = datetime.strptime(start_time, "%Y-%m-%dT%H:%M") if start_time < datetime.utcnow(): raise CumulusCIException("Start time cannot be in the past") else: # delay a bit to allow for review delay_minutes = 5 self.logger.warning( "Scheduling push for %d minutes from now", delay_minutes ) start_time = datetime.utcnow() + timedelta(minutes=delay_minutes) self.request_id, num_scheduled_orgs = self.push.create_push_request( version, orgs, start_time ) self.return_values["request_id"] = self.request_id if num_scheduled_orgs > 1000: sleep_time_s = 30 self.logger.info( "Delaying {} seconds to allow all jobs to initialize".format( sleep_time_s ) ) time.sleep(sleep_time_s) elif num_scheduled_orgs == 0: self.logger.warning("Canceling push request with 0 orgs") self.push.cancel_push_request return self.logger.info("Setting status to Pending to queue execution.") self.logger.info("The push upgrade will start at UTC {}".format(start_time)) # Run the job self.logger.info(self.push.run_push_request(self.request_id)) self.logger.info( "Push Request {} is queued for execution.".format(self.request_id) ) # Report the status if start time is less than 1 minute from now if start_time - datetime.utcnow() < timedelta(minutes=1): self._report_push_status(self.request_id) else: self.logger.info("Exiting early since request is in the future")
class SchedulePushOrgList(BaseSalesforcePushTask): task_options = { "orgs": { "description": "The path to a file containing one OrgID per line.", "required": True, }, "version": { "description": "The managed package version to push", "required": True, }, "namespace": { "description": ( "The managed package namespace to push." + " Defaults to project__package__namespace." ) }, "start_time": { "description": ( "Set the start time (UTC) to queue a future push." + " Ex: 2016-10-19T10:00" ) }, "batch_size": { "description": ( "Break pull requests into batches of this many orgs." + " Defaults to 200." ) }, } def _init_task(self): super(SchedulePushOrgList, self)._init_task() self.push = SalesforcePushApi(self.sf, self.logger, self.options["batch_size"]) def _init_options(self, kwargs): super(SchedulePushOrgList, self)._init_options(kwargs) # Set the namespace option to the value from cumulusci.yml if not # already set if not "namespace" in self.options: self.options["namespace"] = self.project_config.project__package__namespace if not "batch_size" in self.options: self.options["batch_size"] = 200 def _get_orgs(self): return self._load_orgs_file(self.options.get("orgs")) def _run_task(self): orgs = self._get_orgs() package = self._get_package(self.options.get("namespace")) version = self._get_version(package, self.options.get("version")) start_time = self.options.get("start_time") if start_time: if start_time.lower() == "now": start_time = datetime.utcnow() + timedelta(seconds=5) else: start_time = datetime.strptime(start_time, "%Y-%m-%dT%H:%M") if start_time < datetime.utcnow(): raise CumulusCIException("Start time cannot be in the past") else: # delay a bit to allow for review delay_minutes = 5 self.logger.warn("Scheduling push for %d minutes from now", delay_minutes) start_time = datetime.utcnow() + timedelta(minutes=delay_minutes) self.request_id, num_scheduled_orgs = self.push.create_push_request( version, orgs, start_time ) self.return_values["request_id"] = self.request_id if num_scheduled_orgs > 1000: sleep_time_s = 30 self.logger.info( "Delaying {} seconds to allow all jobs to initialize".format( sleep_time_s ) ) time.sleep(sleep_time_s) elif num_scheduled_orgs == 0: self.logger.warn("Canceling push request with 0 orgs") self.push.cancel_push_request return self.logger.info("Setting status to Pending to queue execution.") self.logger.info("The push upgrade will start at UTC {}".format(start_time)) # Run the job self.logger.info(self.push.run_push_request(self.request_id)) self.logger.info( "Push Request {} is queued for execution.".format(self.request_id) ) # Report the status if start time is less than 1 minute from now if start_time - datetime.utcnow() < timedelta(minutes=1): self._report_push_status(self.request_id) else: self.logger.info("Exiting early since request is in the future")
class BaseSalesforcePushTask(BaseSalesforceApiTask): completed_statuses = ['Succeeded','Failed','Cancelled'] def _init_task(self): super(BaseSalesforcePushTask, self)._init_task() self.push = SalesforcePushApi(self.sf, self.logger) def _parse_version(self, version): # Parse the version number string major = None minor = None patch = None build = None state = 'Released' version_parts = version.split('.') if len(version_parts) >= 1: major = version_parts[0] if len(version_parts) == 2: minor = version_parts[1] if minor.find('Beta') != -1: state = 'Beta' minor, build = minor.replace(' (Beta ',',').replace(')','').split(',') if len(version_parts) > 2: minor = version_parts[1] patch = version_parts[2] if patch.find('Beta') != -1: state = 'Beta' patch, build = minor.replace(' (Beta ',',').replace(')','').split(',') return { 'major': major, 'minor': minor, 'patch': patch, 'build': build, 'state': state, } def _get_version(self, package, version): version_info = self._parse_version(version) version_where = "ReleaseState = '{}' AND MajorVersion = {} AND MinorVersion = {}".format( version_info['state'], version_info['major'], version_info['minor'], ) if version_info.get('patch'): version_where += " AND PatchVersion = {}".format(version_info['patch']) if version_info['state'] == 'Beta' and version_info.get('build'): version_where += " AND BuildNumber = {}".format(version_info['build']) version = package.get_package_version_objs(version_where, limit=1) if not version: raise PushApiObjectNotFound('PackageVersion not found. Namespace = {}, Version Info = {}'.format(package.namespace, version_info)) return version[0] def _get_package(self, namespace): package = self.push.get_package_objs( "NamespacePrefix = '{}'".format(namespace), limit=1 ) if not package: raise PushApiObjectNotFound('The package with namespace {} was not found'.format(namespace)) return package[0] def _load_orgs_file(self, path): f_orgs = open(path, 'r') orgs = [] for org in f_orgs: orgs.append(org.strip()) return orgs def _report_push_status(self, request_id): default_where = {'PackagePushRequest': "Id = '{}'".format(request_id)} # Create a new PushAPI instance with different settings than self.push self.push_report = SalesforcePushApi( self.sf, self.logger, lazy = ['subscribers','jobs'], default_where = default_where, ) # Get the push request push_request = self.push_report.get_push_request_objs("Id = '{}'".format(request_id), limit=1) if not push_request: raise PushApiObjectNotFound('Push Request {} was not found'.format(push_request)) push_request = push_request[0] # Check if the request is complete interval = 10 if push_request.status not in self.completed_statuses: self.logger.info( 'Push request is not yet complete. Polling for status every {} seconds until completion...'.format(interval) ) # Loop waiting for request completion i = 0 while push_request.status not in self.completed_statuses: if i == 10: self.logger.info('This is taking a while! Polling every 60 seconds...') interval = 60 time.sleep(interval) # Clear the method level cache on get_push_requests and get_push_request_objs self.push_report.get_push_requests.cache.clear() self.push_report.get_push_request_objs.cache.clear() # Get the push_request again push_request = self.push_report.get_push_request_objs("Id = '{}'".format(request_id), limit=1)[0] self.logger.info(push_request.status) i += 1 failed_jobs = [] success_jobs = [] cancelled_jobs = [] jobs = push_request.get_push_job_objs() for job in jobs: if job.status == 'Failed': failed_jobs.append(job) elif job.status == 'Succeeded': success_jobs.append(job) elif job.status == 'Cancelled': cancelled_jobs.append(job) self.logger.info( "Push complete: {} succeeded, {} failed, {} cancelled".format( len(success_jobs), len(failed_jobs), len(cancelled_jobs), ) ) failed_by_error = {} for job in failed_jobs: errors = job.get_push_error_objs() for error in errors: error_key = (error.error_type, error.title, error.message, error.details) if error_key not in failed_by_error: failed_by_error[error_key] = [] failed_by_error[error_key].append(error) if failed_jobs: self.logger.info("-----------------------------------") self.logger.info("Failures by error type") self.logger.info("-----------------------------------") for key, errors in failed_by_error.items(): self.logger.info(" ") self.logger.info("{} failed with...".format(len(errors))) self.logger.info(" Error Type = {}".format(key[0])) self.logger.info(" Title = {}".format(key[1])) self.logger.info(" Message = {}".format(key[2])) self.logger.info(" Details = {}".format(key[3]))
def sf_push_api(self): return SalesforcePushApi(mock.Mock(), mock.Mock()) # sf # logger
def _init_task(self): super(BaseSalesforcePushTask, self)._init_task() self.push = SalesforcePushApi(self.sf, self.logger)
class BaseSalesforcePushTask(BaseSalesforceApiTask): completed_statuses = ["Succeeded", "Failed", "Canceled"] api_version = "38.0" def _init_task(self): super(BaseSalesforcePushTask, self)._init_task() self.push = SalesforcePushApi(self.sf, self.logger) def _parse_version(self, version): # Parse the version number string major = None minor = None patch = None build = None state = "Released" version_parts = version.split(".") if len(version_parts) >= 1: major = version_parts[0] if len(version_parts) == 2: minor = version_parts[1] if minor.find("Beta") != -1: state = "Beta" minor, build = minor.replace(" (Beta ", ",").replace(")", "").split(",") if len(version_parts) > 2: minor = version_parts[1] patch = version_parts[2] if patch.find("Beta") != -1: state = "Beta" patch, build = minor.replace(" (Beta ", ",").replace(")", "").split(",") return { "major": major, "minor": minor, "patch": patch, "build": build, "state": state, } def _get_version(self, package, version): version_info = self._parse_version(version) version_where = ( "ReleaseState = '{}'".format(version_info["state"]) + " AND MajorVersion = {}".format(version_info["major"]) + " AND MinorVersion = {}".format(version_info["minor"]) ) if version_info.get("patch"): version_where += " AND PatchVersion = {}".format(version_info["patch"]) if version_info["state"] == "Beta" and version_info.get("build"): version_where += " AND BuildNumber = {}".format(version_info["build"]) version = package.get_package_version_objs(version_where, limit=1) if not version: raise PushApiObjectNotFound( "PackageVersion not found." + " Namespace = {}, Version Info = {}".format( package.namespace, version_info ) ) return version[0] def _get_package(self, namespace): package = self.push.get_package_objs( "NamespacePrefix = '{}'".format(namespace), limit=1 ) if not package: raise PushApiObjectNotFound( "The package with namespace {} was not found".format(namespace) ) return package[0] def _load_orgs_file(self, path): orgs = [] with open(path, "r") as f: for line in f: if line.isspace(): continue orgs.append(line.split()[0]) return orgs def _report_push_status(self, request_id): default_where = {"PackagePushRequest": "Id = '{}'".format(request_id)} # Create a new PushAPI instance with different settings than self.push self.push_report = SalesforcePushApi( self.sf, self.logger, lazy=["subscribers", "jobs"], default_where=default_where, ) # Get the push request push_request = self.push_report.get_push_request_objs( "Id = '{}'".format(request_id), limit=1 ) if not push_request: raise PushApiObjectNotFound( "Push Request {} was not found".format(push_request) ) push_request = push_request[0] # Check if the request is complete interval = 10 if push_request.status not in self.completed_statuses: self.logger.info( "Push request is not yet complete." + " Polling for status every {} seconds until completion".format( interval ) ) # Loop waiting for request completion i = 0 while push_request.status not in self.completed_statuses: if i == 10: self.logger.info("This is taking a while! Polling every 60 seconds") interval = 60 time.sleep(interval) # Clear the method level cache on get_push_requests and # get_push_request_objs self.push_report.get_push_requests.cache.clear() self.push_report.get_push_request_objs.cache.clear() # Get the push_request again push_request = self.push_report.get_push_request_objs( "Id = '{}'".format(request_id), limit=1 )[0] self.logger.info(push_request.status) i += 1 failed_jobs = [] success_jobs = [] canceled_jobs = [] jobs = push_request.get_push_job_objs() for job in jobs: if job.status == "Failed": failed_jobs.append(job) elif job.status == "Succeeded": success_jobs.append(job) elif job.status == "Canceled": canceled_jobs.append(job) self.logger.info( "Push complete: {} succeeded, {} failed, {} canceled".format( len(success_jobs), len(failed_jobs), len(canceled_jobs) ) ) failed_by_error = {} for job in failed_jobs: errors = job.get_push_error_objs() for error in errors: error_key = ( error.error_type, error.title, error.message, error.details, ) if error_key not in failed_by_error: failed_by_error[error_key] = [] failed_by_error[error_key].append(error) if failed_jobs: self.logger.info("-----------------------------------") self.logger.info("Failures by error type") self.logger.info("-----------------------------------") for key, errors in failed_by_error.items(): self.logger.info(" ") self.logger.info("{} failed with...".format(len(errors))) self.logger.info(" Error Type = {}".format(key[0])) self.logger.info(" Title = {}".format(key[1])) self.logger.info(" Message = {}".format(key[2])) self.logger.info(" Details = {}".format(key[3]))
class SchedulePushOrgList(BaseSalesforcePushTask): task_options = { 'orgs': { 'description': 'The path to a file containing one OrgID per line.', 'required': True, }, 'version': { 'description': 'The managed package version to push', 'required': True, }, 'namespace': { 'description': ('The managed package namespace to push.' + ' Defaults to project__package__namespace.') }, 'start_time': { 'description': ('Set the start time (UTC) to queue a future push.' + ' Ex: 2016-10-19T10:00') }, 'batch_size': { 'description': ('Break pull requests into batches of this many orgs.' + ' Defaults to 200.') }, } def _init_task(self): super(SchedulePushOrgList, self)._init_task() self.push = SalesforcePushApi( self.sf, self.logger, self.options['batch_size'], ) def _init_options(self, kwargs): super(SchedulePushOrgList, self)._init_options(kwargs) # Set the namespace option to the value from cumulusci.yml if not # already set if not 'namespace' in self.options: self.options['namespace'] = ( self.project_config.project__package__namespace) if not 'batch_size' in self.options: self.options['batch_size'] = 200 def _get_orgs(self): return self._load_orgs_file(self.options.get('orgs')) def _run_task(self): orgs = self._get_orgs() package = self._get_package(self.options.get('namespace')) version = self._get_version(package, self.options.get('version')) start_time = self.options.get('start_time') if start_time: if start_time.lower() == 'now': start_time = datetime.utcnow() + timedelta(seconds=5) else: start_time = datetime.strptime(start_time, '%Y-%m-%dT%H:%M') if start_time < datetime.utcnow(): raise CumulusCIException('Start time cannot be in the past') else: # delay a bit to allow for review delay_minutes = 5 self.logger.warn( 'Scheduling push for %d minutes from now', delay_minutes, ) start_time = datetime.utcnow() + timedelta(minutes=delay_minutes) self.request_id, num_scheduled_orgs = self.push.create_push_request( version, orgs, start_time, ) self.return_values['request_id'] = self.request_id if num_scheduled_orgs > 1000: sleep_time_s = 30 self.logger.info( 'Delaying {} seconds to allow all jobs to initialize'.format( sleep_time_s)) time.sleep(sleep_time_s) elif num_scheduled_orgs == 0: self.logger.warn('Canceling push request with 0 orgs') self.push.cancel_push_request return self.logger.info('Setting status to Pending to queue execution.') self.logger.info( 'The push upgrade will start at UTC {}'.format(start_time)) # Run the job self.logger.info(self.push.run_push_request(self.request_id)) self.logger.info('Push Request {} is queued for execution.'.format( self.request_id)) # Report the status if start time is less than 1 minute from now if start_time - datetime.utcnow() < timedelta(minutes=1): self._report_push_status(self.request_id) else: self.logger.info('Exiting early since request is in the future')
def _get_orgs(self): subscriber_where = self.options.get('subscriber_where') default_where = { 'PackageSubscriber': ("OrgStatus = 'Active' AND InstalledStatus = 'i'") } if subscriber_where: default_where['PackageSubscriber'] += ' AND ({})'.format( subscriber_where) push_api = SalesforcePushApi( self.sf, self.logger, default_where=default_where.copy(), ) package = self._get_package(self.options.get('namespace')) version = self._get_version(package, self.options.get('version')) min_version = self.options.get('min_version') if min_version: min_version = self._get_version( package, self.options.get('min_version'), ) orgs = [] if min_version: # If working with a range of versions, use an inclusive search versions = version.get_older_released_version_objs( greater_than_version=min_version) included_versions = [] for include_version in versions: included_versions.append(str(include_version.sf_id)) if not included_versions: raise ValueError( 'No versions found between version id {} and {}'.format( version.version_number, min_version.version_number)) # Query orgs for each version in the range individually to avoid # query timeout errors with querying multiple versions for included_version in included_versions: # Clear the get_subscribers method cache before each call push_api.get_subscribers.cache.clear() push_api.default_where['PackageSubscriber'] = ( "{} AND MetadataPackageVersionId = '{}'".format( default_where['PackageSubscriber'], included_version, )) for subscriber in push_api.get_subscribers(): orgs.append(subscriber['OrgKey']) else: # If working with a specific version rather than a range, use an # exclusive search. # Add exclusion of all orgs running on newer releases newer_versions = version.get_newer_released_version_objs() excluded_versions = [str(version.sf_id)] for newer in newer_versions: excluded_versions.append(str(newer.sf_id)) if len(excluded_versions) == 1: push_api.default_where['PackageSubscriber'] += ( " AND MetadataPackageVersionId != '{}'".format( excluded_versions[0], )) else: push_api.default_where['PackageSubscriber'] += ( " AND MetadataPackageVersionId NOT IN {}".format( "('" + "','".join(excluded_versions) + "')")) for subscriber in push_api.get_subscribers(): orgs.append(subscriber['OrgKey']) return orgs
def sf_push_api(): return SalesforcePushApi(sf=mock.Mock(), logger=mock.Mock())