def update_subsequent_requests(self, request, values): """ Update all subsequent requests """ request_db = Database('requests') prepid = request.get_prepid() query = f'input.request={prepid}' requests = request_db.query(query) self.logger.info('Found %s subsequent requests for %s: %s', len(requests), prepid, [r['prepid'] for r in requests]) for request_json in requests: if request_json.get('status') not in ('new', 'approved'): continue request_prepid = request_json.get('prepid', '') try: subsequent_request = self.get(request_prepid) for key, value in values.items(): subsequent_request.set(key, value) self.update(subsequent_request.get_json()) except Exception as ex: self.logger.error('Error updating subsequent request %s: %s', request_prepid, ex)
def move_to_done(database_auth, port): """ Try to move all submitted RelVals to next status """ Database.set_database_name('relval') Database.set_credentials_file(database_auth) connection = http.client.HTTPConnection('localhost', port=port, timeout=300) headers = { 'Content-Type': 'application/json', 'Adfs-Login': '******', 'Adfs-Group': 'cms-pdmv-serv' } relval_db = Database('relvals') relvals = [{}] page = 0 while relvals: relvals = relval_db.query(query_string='status=submitted', page=page) page += 1 for relval in relvals: print(relval['prepid']) connection.request('POST', '/api/relvals/next_status', json.dumps(relval), headers=headers) response = connection.getresponse() response_text = json.loads(response.read())['message'] print(' %s %s' % (response.code, response_text))
def submit_subsequent_requests(self, request): """ Submit all requests that have given request as input """ request_db = Database('requests') prepid = request.get_prepid() query = f'input.request={prepid}' subsequent_requests = request_db.query(query) self.logger.info('Found %s subsequent requests for %s: %s', len(subsequent_requests), prepid, [r['prepid'] for r in subsequent_requests]) for subsequent_request_json in subsequent_requests: subsequent_request_prepid = subsequent_request_json.get( 'prepid', '') try: subsequent_request = self.get(subsequent_request_prepid) self.update_input_dataset(subsequent_request) if subsequent_request.get('status') == 'new': self.next_status(subsequent_request) if subsequent_request.get('status') == 'approved': self.next_status(subsequent_request) except Exception as ex: self.logger.error('Error moving %s to next status: %s', subsequent_request_prepid, ex)
def after_update(self, old_obj, new_obj, changed_values): self.logger.info('Changed values: %s', changed_values) if 'workflow_name' in changed_values: new_relval = self.create(new_obj.get_json()) old_prepid = old_obj.get_prepid() new_prepid = new_relval.get_prepid() new_relval.set('history', old_obj.get('history')) new_relval.add_history('rename', [old_prepid, new_prepid], None) relvals_db = Database('relvals') relvals_db.save(new_relval.get_json()) self.logger.info('Created %s as rename of %s', new_prepid, old_prepid) new_obj.set('prepid', new_prepid) # Update the ticket... tickets_db = Database('tickets') tickets = tickets_db.query( f'created_relvals={old_obj.get_prepid()}') self.logger.debug(json.dumps(tickets, indent=2)) for ticket_json in tickets: ticket_prepid = ticket_json['prepid'] with self.locker.get_lock(ticket_prepid): ticket_json = tickets_db.get(ticket_prepid) ticket = Ticket(json_input=ticket_json) created_relvals = ticket.get('created_relvals') if old_prepid in created_relvals: created_relvals.remove(old_prepid) created_relvals.append(new_prepid) ticket.set('created_relvals', created_relvals) ticket.add_history('rename', [old_prepid, new_prepid], None) tickets_db.save(ticket.get_json()) self.delete(old_obj.get_json())
def after_delete(self, obj): prepid = obj.get_prepid() tickets_db = Database('tickets') tickets = tickets_db.query(f'created_relvals={prepid}') self.logger.debug(json.dumps(tickets, indent=2)) for ticket_json in tickets: ticket_prepid = ticket_json['prepid'] with self.locker.get_lock(ticket_prepid): ticket_json = tickets_db.get(ticket_prepid) ticket = Ticket(json_input=ticket_json) created_relvals = ticket.get('created_relvals') if prepid in created_relvals: created_relvals.remove(prepid) ticket.set('created_relvals', created_relvals) ticket.add_history('remove_relval', prepid, None) tickets_db.save(ticket.get_json())
def check_for_delete(self, obj): if obj.get('status') != 'new': raise Exception( 'Request must be in status "new" before it is deleted') requests_db = Database('requests') prepid = obj.get_prepid() subsequent_requests_query = f'input.request={prepid}' subsequent_requests = requests_db.query(subsequent_requests_query) if subsequent_requests: subsequent_requests_prepids = ', '.join( [r['prepid'] for r in subsequent_requests]) raise Exception( f'Request cannot be deleted because it is input request' f'for {subsequent_requests_prepids}. Delete these requests first' ) return True
""" Script to add run list to relval steps """ import sys import os.path import os sys.path.append(os.path.abspath(os.path.pardir)) from core_lib.database.database import Database Database.set_credentials_file(os.getenv('DB_AUTH')) Database.set_database_name('relval') relvals_database = Database('relvals') total_relvals = relvals_database.get_count() print('Total relvals: %s' % (total_relvals)) for index, item in enumerate(relvals_database.query(limit=total_relvals)): print('Processing entry %s/%s %s' % (index + 1, total_relvals, item.get('prepid', '<no-id>'))) for step in item['steps']: step['input']['run'] = step['input'].get('run', []) relvals_database.save(item) print('Done')
subcampaign_db = Database('subcampaigns') old_ticket_db = Database('subcampaign_tickets') new_ticket_db = Database('tickets') total_subcampaigns = subcampaign_db.get_count() total_requests = request_db.get_count() total_old_tickets = old_ticket_db.get_count() total_new_tickets = new_ticket_db.get_count() print('Requests: %s' % (total_requests)) print('Subcampaigns: %s' % (total_subcampaigns)) print('(Old) subcampaign tickets: %s' % (total_old_tickets)) print('(New) tickets: %s' % (total_new_tickets)) for index, subcampaign in enumerate( subcampaign_db.query(limit=total_subcampaigns)): print('Processing subcampaign %s/%s %s' % (index + 1, total_subcampaigns, subcampaign['prepid'])) subcampaign.pop('_rev', None) subcampaign.pop('step', None) subcampaign_db.save(subcampaign) for index, request in enumerate(request_db.query(limit=total_requests)): print('Processing request %s/%s %s' % (index + 1, total_requests, request['prepid'])) request.pop('_rev', None) request.pop('step', None) if 'input_dataset' in request: request['input'] = { 'dataset': request.pop('input_dataset'), 'request': ''
sys.path.append(os.path.abspath(os.path.pardir)) from core_lib.database.database import Database Database.set_credentials_file(os.getenv('DB_AUTH')) Database.set_database_name('rereco') request_db = Database('requests') subcampaign_db = Database('subcampaigns') total_subcampaigns = subcampaign_db.get_count() total_requests = request_db.get_count() print('Requests: %s' % (total_requests)) print('Subcampaigns: %s' % (total_subcampaigns)) for index, subcampaign in enumerate(subcampaign_db.query(limit=total_subcampaigns)): print('Processing subcampaign %s/%s %s' % (index + 1, total_subcampaigns, subcampaign['prepid'])) subcampaign.pop('scram_arch', None) subcampaign_db.save(subcampaign) for index, request in enumerate(request_db.query(limit=total_requests)): print('Processing request %s/%s %s' % (index + 1, total_requests, request['prepid'])) request.pop('scram_arch', None) request_db.save(request) total_subcampaigns = subcampaign_db.get_count() total_requests = request_db.get_count()
def get(self): """ Perform a search """ args = flask.request.args.to_dict() if args is None: args = {} db_name = args.pop('db_name', None) page = int(args.pop('page', 0)) limit = int(args.pop('limit', 20)) sort = args.pop('sort', None) sort_asc = args.pop('sort_asc', None) # Special cases from_ticket = args.pop('ticket', None) if db_name == 'relvals' and from_ticket: ticket_database = Database('tickets') tickets = ticket_database.query( query_string=f'prepid={from_ticket}', limit=100, ignore_case=True) created_relvals = [] for ticket in tickets: created_relvals.extend(ticket['created_relvals']) created_relvals = ','.join(created_relvals) prepid_query = args.pop('prepid', '') args['prepid'] = ('%s,%s' % (prepid_query, created_relvals)).strip(',') # Sorting logic: by default sort dsc by cration time if sort is None: sort = 'created_on' if sort == 'created_on' and sort_asc is None: sort_asc = False if sort_asc is None: sort_asc = True limit = max(1, min(limit, 500)) sort_asc = str(sort_asc).lower() == 'true' query_string = '&&'.join(['%s=%s' % (pair) for pair in args.items()]) database = Database(db_name) query_string = database.build_query_with_types(query_string, self.classes[db_name]) results, total_rows = database.query_with_total_rows( query_string=query_string, page=page, limit=limit, sort_attr=sort, sort_asc=sort_asc, ignore_case=True) return self.output_text({ 'response': { 'results': results, 'total_rows': total_rows }, 'success': True, 'message': '' })
import sys import os.path import os sys.path.append(os.path.abspath(os.path.pardir)) from core_lib.database.database import Database Database.set_credentials_file(os.getenv('DB_AUTH')) Database.set_database_name('relval') database = Database('relvals') total_entries = database.get_count() print('Total entries: %s' % (total_entries)) for index, item in enumerate(database.query(limit=total_entries)): print('Processing entry %s/%s %s' % (index + 1, total_entries, item.get('prepid', '<no-id>'))) item['job_dict_overwrite'] = {} database.save(item) print('Done')
def update_workflows(self, request): """ Update computing workflows from Stats2 """ prepid = request.get_prepid() request_db = Database('requests') with self.locker.get_lock(prepid): request_json = request_db.get(prepid) request = Request(json_input=request_json) stats_conn = ConnectionWrapper(host='vocms074.cern.ch', port=5984, https=False, keep_open=True) stats_workflows = stats_conn.api( 'GET', f'/requests/_design/_designDoc/_view/prepids?key="{prepid}"&include_docs=True' ) stats_workflows = json.loads(stats_workflows) stats_workflows = [x['doc'] for x in stats_workflows['rows']] existing_workflows = [x['name'] for x in request.get('workflows')] stats_workflows = [x['RequestName'] for x in stats_workflows] all_workflow_names = list( set(existing_workflows) | set(stats_workflows)) self.logger.info('All workflows of %s are %s', prepid, ', '.join(all_workflow_names)) all_workflows = {} for workflow_name in all_workflow_names: workflow = stats_conn.api('GET', f'/requests/{workflow_name}') if not workflow: raise Exception( f'Could not find {workflow_name} in Stats2') workflow = json.loads(workflow) if not workflow.get('RequestName'): raise Exception( f'Could not find {workflow_name} in Stats2') if workflow.get('RequestType', '').lower() == 'resubmission': continue all_workflows[workflow_name] = workflow self.logger.info('Fetched workflow %s', workflow_name) stats_conn.close() output_datasets = self.__get_output_datasets( request, all_workflows) new_workflows = self.__pick_workflows(all_workflows, output_datasets) all_workflow_names = [x['name'] for x in new_workflows] for new_workflow in reversed(new_workflows): completed_events = -1 for output_dataset in new_workflow.get('output_datasets', []): if output_datasets and output_dataset[ 'name'] == output_datasets[-1]: completed_events = output_dataset['events'] break if completed_events != -1: request.set('completed_events', completed_events) break if all_workflow_names: newest_workflow = all_workflows[all_workflow_names[-1]] if 'RequestPriority' in newest_workflow: request.set('priority', newest_workflow['RequestPriority']) if 'TotalEvents' in newest_workflow: request.set('total_events', max(0, newest_workflow['TotalEvents'])) request.set('output_datasets', output_datasets) request.set('workflows', new_workflows) request_db.save(request.get_json()) if output_datasets: subsequent_requests = request_db.query( f'input.request={prepid}') self.logger.info('Found %s subsequent requests for %s: %s', len(subsequent_requests), prepid, [r['prepid'] for r in subsequent_requests]) for subsequent_request_json in subsequent_requests: subsequent_request_prepid = subsequent_request_json.get( 'prepid') self.update_input_dataset( self.get(subsequent_request_prepid)) return request
def move_relvals_to_submitting(self, relvals): """ Try to add RelVals to submission queue and get sumbitted """ results = [] dataset_access_types = self.get_dataset_access_types(relvals) for relval in relvals: prepid = relval.get_prepid() with self.locker.get_nonblocking_lock(prepid): batch_name = relval.get('batch_name') cmssw_release = relval.get('cmssw_release') relval_db = Database('relvals') # Make sure all datasets are VALID in DBS steps = relval.get('steps') for step in steps: if step.get_step_type() == 'input_file': dataset = step.get('input')['dataset'] elif step.get('driver')['pileup_input']: dataset = step.get('driver')['pileup_input'] else: continue dataset = dataset[dataset.index('/'):] access_type = dataset_access_types[dataset] if access_type.lower() != 'valid': raise Exception( f'{dataset} type is {access_type}, it must be VALID' ) # Create or find campaign timestamp # Threshold in seconds threshold = 3600 locker_key = f'move-relval-to-submitting-{cmssw_release}__{batch_name}' with self.locker.get_lock(locker_key): now = int(time.time()) # Get RelVal with newest timestamp in this campaign (CMSSW + Batch Name) db_query = f'cmssw_release={cmssw_release}&&batch_name={batch_name}' relvals_with_timestamp = relval_db.query( db_query, limit=1, sort_attr='campaign_timestamp', sort_asc=False) newest_timestamp = 0 if relvals_with_timestamp: newest_timestamp = relvals_with_timestamp[0].get( 'campaign_timestamp', 0) self.logger.info( 'Newest timestamp for %s__%s is %s (%s), threshold is %s', cmssw_release, batch_name, newest_timestamp, (newest_timestamp - now), threshold) if newest_timestamp == 0 or newest_timestamp < now - threshold: newest_timestamp = now self.logger.info( 'Campaign timestamp for %s__%s will be set to %s', cmssw_release, batch_name, newest_timestamp) relval.set('campaign_timestamp', newest_timestamp) self.update_status(relval, 'submitting') RequestSubmitter().add(relval, self) results.append(relval) return results