def __init__(self, config): # first grab the admin list self.adminList = [] if 'admin-users' in config: tokens = config['admin-users'].split(',') for t in tokens: if t.strip(): self.adminList.append(t.strip()) if not self.adminList: warnings.warn('no "admin-users" are set in config of CatalogController.') # make sure the minimal mongo settings are in place if 'mongodb-host' not in config: raise ValueError('"mongodb-host" config variable must be defined to start a CatalogController!') if 'mongodb-database' not in config: raise ValueError('"mongodb-database" config variable must be defined to start a CatalogController!') # give warnings if no mongo user information is set if 'mongodb-user' not in config: warnings.warn('"mongodb-user" is not set in config of CatalogController.') config['mongodb-user']='' config['mongodb-pwd']='' if 'mongodb-pwd' not in config: warnings.warn('"mongodb-pwd" is not set in config of CatalogController.') config['mongodb-pwd']='' # instantiate the mongo client self.db = MongoCatalogDBI( config['mongodb-host'], config['mongodb-database'], config['mongodb-user'], config['mongodb-pwd']) # check for the temp directory and make sure it exists if 'temp-dir' not in config: raise ValueError('"temp-dir" config variable must be defined to start a CatalogController!') self.temp_dir = config['temp-dir'] if not os.path.exists(self.temp_dir): raise ValueError('"temp-dir" does not exist! It is required for registration to work!') if not os.path.exists(self.temp_dir): raise ValueError('"temp-dir" does not exist! Space is required for registration to work!') if not os.access(self.temp_dir, os.W_OK): raise ValueError('"temp-dir" not writable! Writable space is required for registration to work!') if 'docker-base-url' not in config: raise ValueError('"docker-base-url" config variable must be defined to start a CatalogController!') self.docker_base_url = config['docker-base-url'] print(self.docker_base_url) if 'docker-registry-host' not in config: raise ValueError('"docker-registry-host" config variable must be defined to start a CatalogController!') self.docker_registry_host = config['docker-registry-host'] print(self.docker_registry_host) if 'nms-url' not in config: raise ValueError('"nms-url" config variable must be defined to start a CatalogController!') self.nms_url = config['nms-url'] if 'nms-admin-user' not in config: raise ValueError('"nms-admin-user" config variable must be defined to start a CatalogController!') self.nms_admin_user = config['nms-admin-user'] if 'nms-admin-psswd' not in config: raise ValueError('"nms-admin-psswd" config variable must be defined to start a CatalogController!') self.nms_admin_psswd = config['nms-admin-psswd']
class CatalogController: def __init__(self, config): # first grab the admin list self.adminList = [] if 'admin-users' in config: tokens = config['admin-users'].split(',') for t in tokens: if t.strip(): self.adminList.append(t.strip()) if not self.adminList: warnings.warn('no "admin-users" are set in config of CatalogController.') # make sure the minimal mongo settings are in place if 'mongodb-host' not in config: raise ValueError('"mongodb-host" config variable must be defined to start a CatalogController!') if 'mongodb-database' not in config: raise ValueError('"mongodb-database" config variable must be defined to start a CatalogController!') # give warnings if no mongo user information is set if 'mongodb-user' not in config: warnings.warn('"mongodb-user" is not set in config of CatalogController.') config['mongodb-user']='' config['mongodb-pwd']='' if 'mongodb-pwd' not in config: warnings.warn('"mongodb-pwd" is not set in config of CatalogController.') config['mongodb-pwd']='' # instantiate the mongo client self.db = MongoCatalogDBI( config['mongodb-host'], config['mongodb-database'], config['mongodb-user'], config['mongodb-pwd']) # check for the temp directory and make sure it exists if 'temp-dir' not in config: raise ValueError('"temp-dir" config variable must be defined to start a CatalogController!') self.temp_dir = config['temp-dir'] if not os.path.exists(self.temp_dir): raise ValueError('"temp-dir" does not exist! It is required for registration to work!') if not os.path.exists(self.temp_dir): raise ValueError('"temp-dir" does not exist! Space is required for registration to work!') if not os.access(self.temp_dir, os.W_OK): raise ValueError('"temp-dir" not writable! Writable space is required for registration to work!') if 'docker-base-url' not in config: raise ValueError('"docker-base-url" config variable must be defined to start a CatalogController!') self.docker_base_url = config['docker-base-url'] print(self.docker_base_url) if 'docker-registry-host' not in config: raise ValueError('"docker-registry-host" config variable must be defined to start a CatalogController!') self.docker_registry_host = config['docker-registry-host'] print(self.docker_registry_host) if 'nms-url' not in config: raise ValueError('"nms-url" config variable must be defined to start a CatalogController!') self.nms_url = config['nms-url'] if 'nms-admin-user' not in config: raise ValueError('"nms-admin-user" config variable must be defined to start a CatalogController!') self.nms_admin_user = config['nms-admin-user'] if 'nms-admin-psswd' not in config: raise ValueError('"nms-admin-psswd" config variable must be defined to start a CatalogController!') self.nms_admin_psswd = config['nms-admin-psswd'] def register_repo(self, params, username, token): if 'git_url' not in params: raise ValueError('git_url not defined, but is required for registering a repository') git_url = params['git_url'] if not bool(urlparse(git_url).netloc): raise ValueError('The git url provided is not a valid URL.') timestamp = int((datetime.utcnow() - datetime.utcfromtimestamp(0)).total_seconds()*1000) # 0) Make sure the submitter is on the list if not self.is_approved_developer([username])[0]: raise ValueError('You are not an approved developer. Contact us to request approval.') # 1) If the repo does not yet exist, then create it. No additional permission checks needed if not self.db.is_registered(git_url=git_url) : self.db.register_new_module(git_url, username, timestamp) module_details = self.db.get_module_details(git_url=git_url) # 2) If it has already been registered, make sure the user has permissions to update, and # that the module is in a state where it can be registered else: module_details = self.db.get_module_details(git_url=git_url) # 2a) Make sure the user has permission to register this URL if self.has_permission(username,module_details['owners']): # 2b) Make sure the current registration state is either 'complete' or 'error' state = module_details['state'] registration_state = state['registration'] if registration_state == 'complete' or registration_state == 'error': error = self.db.set_module_registration_state(git_url=git_url, new_state='started', last_state=registration_state) if error is not None: # we can fail if the registration state changed when we were first checking to now. This is important # to ensure we only ever kick off one registration thread at a time raise ValueError('Registration failed for git repo ('+git_url+') - registration state was modified before build could begin: '+error) # we know we are the only operation working, so we can clear the dev version and upate the timestamp self.db.update_dev_version({'timestamp':timestamp}, git_url=git_url) else: raise ValueError('Registration already in progress for this git repo ('+git_url+')') else : raise ValueError('You ('+username+') are an approved developer, but do not have permission to register this repo ('+git_url+')') # 3) Ok, kick off the registration thread # - This will check out the repo, attempt to build the image, run some tests, store the image # - If all went well, and during the process, it will update the registration state of the # module and finally update the dev version # - If things failed, it will set the error state, and set an error message. # first set the dev current_release timestamp t = threading.Thread(target=_start_registration, args=(params,timestamp,username,token,self.db, self.temp_dir, self.docker_base_url, self.docker_registry_host, self.nms_url, self.nms_admin_user, self.nms_admin_psswd, module_details)) t.start() # 4) provide the timestamp return timestamp def set_registration_state(self, params, username): # first some error handling if not self.is_admin(username): raise ValueError('You do not have permission to modify the registration state of this module/repo.') params = self.filter_module_or_repo_selection(params) if 'registration_state' not in params: raise ValueError('Update failed - no registration state indicated.') #TODO: possibly check for empty states or that the state is a valid state here #if not params['registration_state'] : error_message = '' if params['registration_state'] == 'error': if 'error_message' not in params: raise ValueError('Update failed - if state is "error", you must also set an "error_message".') if params['error_message']: raise ValueError('Update failed - if state is "error", you must also set an "error_message".') error_message = params['error_message'] else: # then we update the state error = self.db.set_module_registration_state( git_url=params['git_url'], module_name=params['module_name'], new_state=params['registration_state'], error_message=error_message) if error is not None: raise ValueError('Registration failed for git repo ('+git_url+')- some unknown database error: ' + error) def push_dev_to_beta(self, params, username): # first make sure everything exists and we have permissions params = self.filter_module_or_repo_selection(params) module_details = self.db.get_module_details(module_name=params['module_name'],git_url=params['git_url']) # Make sure the submitter is still an approved developer if not self.is_approved_developer([username])[0]: raise ValueError('You are not an approved developer. Contact us to request approval.') if not self.has_permission(username,module_details['owners']): raise ValueError('You do not have permission to modify this module/repo.') # next make sure the state of the module is ok (it must be active, no pending registrations or release requests) if not module_details['state']['active']: raise ValueError('Cannot push dev to beta- module/repo is no longer active.') if module_details['state']['registration'] != 'complete': raise ValueError('Cannot push dev to beta- last registration is in progress or has an error.') if module_details['state']['release_approval'] == 'under_review': raise ValueError('Cannot push dev to beta- last release request of beta is still pending.') # ok, do it. error = self.db.push_dev_to_beta(module_name=params['module_name'],git_url=params['git_url']) if error is not None: raise ValueError('Update operation failed - some unknown database error: '+error) def request_release(self, params, username): # first make sure everything exists and we have permissions params = self.filter_module_or_repo_selection(params) module_details = self.db.get_module_details(module_name=params['module_name'],git_url=params['git_url']) # Make sure the submitter is still an approved developer if not self.is_approved_developer([username])[0]: raise ValueError('You are not an approved developer. Contact us to request approval.') if not self.has_permission(username,module_details['owners']): raise ValueError('You do not have permission to modify this module/repo.') # next make sure the state of the module is ok (it must be active, no pending release requests) if not module_details['state']['active']: raise ValueError('Cannot request release - module/repo is no longer active.') if module_details['state']['release_approval'] == 'under_review': raise ValueError('Cannot request release - last release request of beta is still pending.') # beta version must exist if not module_details['current_versions']['beta']: raise ValueError('Cannot request release - no beta version has been created yet.') # beta version must be different than release version (if release version exists) if module_details['current_versions']['release']: if module_details['current_versions']['beta']['timestamp'] == module_details['current_versions']['release']['timestamp']: raise ValueError('Cannot request release - beta version is identical to released version.') # ok, do it. error = self.db.set_module_release_state( module_name=params['module_name'],git_url=params['git_url'], new_state='under_review', last_state=module_details['state']['release_approval'] ) if error is not None: raise ValueError('Release request failed - some unknown database error.'+error) def list_requested_releases(self): query={'state.release_approval':'under_review'} results=self.db.find_current_versions_and_owners(query) requested_releases = [] for r in results: owners = [] for o in r['owners']: owners.append(o['kb_username']) beta = r['current_versions']['beta'] timestamp = beta['timestamp'] requested_releases.append({ 'module_name':r['module_name'], 'git_url':r['git_url'], 'timestamp':timestamp, 'git_commit_hash':beta['git_commit_hash'], 'git_commit_message':beta['git_commit_message'], 'owners':owners }) return requested_releases def review_release_request(self, review, username): if not self.is_admin(username): raise ValueError('You do not have permission to review a release request.') review = self.filter_module_or_repo_selection(review) module_details = self.db.get_module_details(module_name=review['module_name'],git_url=review['git_url']) if module_details['state']['release_approval'] != 'under_review': raise ValueError('Cannot review request - module/repo is not under review!') if not module_details['state']['active']: raise ValueError('Cannot review request - module/repo is no longer active.') if module_details['state']['release_approval'] != 'under_review': raise ValueError('Cannot review request - module/repo is not under review!') if 'decision' not in review: raise ValueError('Cannot set review - no "decision" was provided!') if not review['decision']: raise ValueError('Cannot set review - no "decision" was provided!') if review['decision']=='denied': if 'review_message' not in review: raise ValueError('Cannot set review - if denied, you must set a "review_message"!') if not review['review_message'].strip(): raise ValueError('Cannot set review - if denied, you must set a "review_message"!') if 'review_message' not in review: review['review_message']='' if review['decision'] not in ['approved','denied']: raise ValueError('Cannot set review - decision must be "approved" or "denied"') # ok, do it. # if the state is approved, then we need to save the beta version over the release version and stash # a new entry. The DBI will handle that for us. (note that concurency issues don't really matter # here because if this is done twice (for instance, before the release_state is set to approved in # the document in the next call) there won't be any problems.) I like nested parentheses. if review['decision']=='approved': error = self.db.push_beta_to_release(module_name=review['module_name'],git_url=review['git_url']) # Now we can update the release state state... error = self.db.set_module_release_state( module_name=review['module_name'],git_url=review['git_url'], new_state=review['decision'], last_state=module_details['state']['release_approval'], review_message=review['review_message'] ) if error is not None: raise ValueError('Release review update failed - some unknown database error. ' + error) def get_module_state(self, params): params = self.filter_module_or_repo_selection(params) return self.db.get_module_state(module_name=params['module_name'],git_url=params['git_url']) def get_module_info(self, params): params = self.filter_module_or_repo_selection(params) details = self.db.get_module_details(module_name=params['module_name'], git_url=params['git_url']) owners = [] for o in details['owners']: owners.append(o['kb_username']) info = { 'module_name': details['module_name'], 'git_url': details['git_url'], 'description': details['info']['description'], 'language': details['info']['language'], 'owners': owners, 'release': details['current_versions']['release'], 'beta': details['current_versions']['beta'], 'dev': details['current_versions']['dev'] } return info def get_version_info(self,params): params = self.filter_module_or_repo_selection(params) current_version = self.db.get_module_current_versions(module_name=params['module_name'], git_url=params['git_url']) if not current_version: return None # TODO: can make this more effecient and flexible by putting in some indicies and doing the query on mongo side # right now, we require a module name / git url, and request specific version based on selectors. in the future # we could, for instance, get all versions that match a particular git commit hash, or timestamp... # If version is in params, it should be one of dev, beta, release if 'version' in params: if params['version'] not in ['dev','beta','release']: raise ValueError('invalid version selection, valid versions are: "dev" | "beta" | "release"') v = current_version[params['version']] # if timestamp or git_commit_hash is given, those need to match as well if 'timestamp' in params: if v['timestamp'] != params['timestamp'] : return None; if 'git_commit_hash' in params: if v['git_commit_hash'] != params['git_commit_hash'] : return None; return v if 'timestamp' in params: # first check in current versions for version in ['dev','beta','release']: if current_version[version]['timestamp'] == params['timestamp']: v = current_version[version] if 'git_commit_hash' in params: if v['git_commit_hash'] != params['git_commit_hash'] : return None; return v # if we get here, we have to look in full history details = self.db.get_module_full_details(module_name=params['module_name'], git_url=params['git_url']) all_versions = details['release_versions'] if str(params['timestamp']) in all_versions: v = all_versions[str(params['timestamp'])] if 'git_commit_hash' in params: if v['git_commit_hash'] != params['git_commit_hash'] : return None; return v return None # if we get here, version and timestamp are not defined, so just look for the commit hash if 'git_commit_hash' in params: # check current versions for version in ['dev','beta','release']: if current_version[version]['git_commit_hash'] == params['git_commit_hash']: v = current_version[version] return v # if we get here, we have to look in full history details = self.db.get_module_full_details(module_name=params['module_name'], git_url=params['git_url']) all_versions = details['release_versions'] for timestamp, v in all_versions.iteritems(): if v['git_commit_hash'] == params['git_commit_hash']: return v return None # didn't get nothing, so return return None def list_released_versions(self, params): params = self.filter_module_or_repo_selection(params) details = self.db.get_module_full_details(module_name=params['module_name'], git_url=params['git_url']) return sorted(details['release_versions'].values(), key= lambda v: v['timestamp']) def is_registered(self,params): if 'git_url' not in params: params['git_url'] = '' if 'module_name' not in params: params['module_name'] = '' if self.db.is_registered(module_name=params['module_name'], git_url=params['git_url']) : return True return False # note: maybe a little too mongo centric, but ok for now... def list_basic_module_info(self,params): query = { 'state.active':True, 'state.released':True } if 'include_disabled' in params: if params['include_disabled']>0: query.pop('state.active',None) if 'include_released' not in params: params['include_released'] = 1 if 'include_unreleased' not in params: params['include_unreleased'] = 0 # figure out release/unreleased options so we can get just the unreleased if needed # default (if none of these matches is to list only released) if params['include_released']<=0 and params['include_unreleased']<=0: return [] # don't include anything... elif params['include_released']<=0 and params['include_unreleased']>0: # minor change that could be removed eventually: check for released=False or missing query.pop('state.released',None) query['$or']=[{'state.released':False},{'state.released':{'$exists':False}}] #query['state.released']=False # include only unreleased (only works if everything has this flag) elif params['include_released']>0 and params['include_unreleased']>0: query.pop('state.released',None) # include everything if 'owners' in params: if params['owners']: # might want to filter out empty strings in the future query['owners.kb_username']={'$in':params['owners']} return self.db.find_basic_module_info(query) def set_module_active_state(self, active, params, username): params = self.filter_module_or_repo_selection(params) if not self.is_admin(username): raise ValueError('Only Admin users can set a module to be active/inactive.') error = self.db.set_module_active_state(active, module_name=params['module_name'], git_url=params['git_url']) if error is not None: raise ValueError('Update operation failed - some unknown database error: '+error) def approve_developer(self, developer, username): if not developer: raise ValueError('No username provided') if not developer.strip(): raise ValueError('No username provided') if not self.is_admin(username): raise ValueError('Only Admin users can approve or revoke developers.') self.db.approve_developer(developer) def revoke_developer(self, developer, username): if not developer: raise ValueError('No username provided') if not developer.strip(): raise ValueError('No username provided') if not self.is_admin(username): raise ValueError('Only Admin users can approve or revoke developers.') self.db.revoke_developer(developer) def is_approved_developer(self, usernames): if not usernames: return [] return self.db.is_approved_developer(usernames) def list_approved_developers(self): dev_list = self.db.list_approved_developers() simple_kbase_dev_list = [] for d in dev_list: simple_kbase_dev_list.append(d['kb_username']) return sorted(simple_kbase_dev_list) def get_build_log(self, timestamp): try: with open(self.temp_dir+'/registration.log.'+str(timestamp)) as log_file: log = log_file.read() except: log = '[log not found - timestamp is invalid or the log has been deleted]' return log def migrate_module_to_new_git_url(self, params, username): if not self.is_admin(username): raise ValueError('Only Admin users can migrate module git urls.') if 'module_name' not in params: raise ValueError('You must specify the "module_name" of the module to modify.') if 'current_git_url' not in params: raise ValueError('You must specify the "current_git_url" of the module to modify.') if 'new_git_url' not in params: raise ValueError('You must specify the "new_git_url" of the module to modify.') if not bool(urlparse(params['new_git_url']).netloc): raise ValueError('The new git url is not a valid URL.') error = self.db.migrate_module_to_new_git_url(params['module_name'],params['current_git_url'],params['new_git_url']) if error is not None: raise ValueError('Update operation failed - some unknown database error: '+error) # Some utility methods def filter_module_or_repo_selection(self, params): if 'git_url' not in params: params['git_url'] = '' if 'module_name' not in params: params['module_name'] = '' if not self.db.is_registered(module_name=params['module_name'], git_url=params['git_url']) : raise ValueError('Operation failed - module/repo is not registered.') return params # always true if the user is in the admin list def has_permission(self, username, owners): if self.is_admin(username): return True for owner in owners: if username == owner['kb_username']: return True return False def is_admin(self, username): if username in self.adminList: return True return False def version(self): return biokbase.catalog.version.CATALOG_VERSION