def move_to_done(database_auth, port): """ Try to move all submitted RelVals to next status """ Database.set_database_name('relval') Database.set_credentials_file(database_auth) connection = http.client.HTTPConnection('localhost', port=port, timeout=300) headers = { 'Content-Type': 'application/json', 'Adfs-Login': '******', 'Adfs-Group': 'cms-pdmv-serv' } relval_db = Database('relvals') relvals = [{}] page = 0 while relvals: relvals = relval_db.query(query_string='status=submitted', page=page) page += 1 for relval in relvals: print(relval['prepid']) connection.request('POST', '/api/relvals/next_status', json.dumps(relval), headers=headers) response = connection.getresponse() response_text = json.loads(response.read())['message'] print(' %s %s' % (response.code, response_text))
def dump_documents(database_auth, output_dir, database_name, collections): """ Dump a list of collections to separate directories """ Database.set_database_name(database_name) Database.set_credentials_file(database_auth) for collection_name in collections: print('Collection %s' % (collection_name)) database = Database(collection_name) collection = database.collection doc_count = collection.count_documents({}) print('Found %s documents' % (doc_count)) documents = [{}] page = 0 limit = 100 collection_path = f'{output_dir}/{collection_name}' os.makedirs(collection_path) while documents: documents = collection.find({}).sort('_id', 1).skip(page * limit).limit(limit) documents = [d for d in documents] if not documents: break file_name = f'{collection_path}/{database_name}_{collection_name}_{page}.json' with open(file_name, 'w') as output_file: json.dump(documents, output_file) print('Page %s done' % (page)) page += 1
def main(): """ Main function: start Flask web server """ parser = argparse.ArgumentParser(description='ReReco Machine') parser.add_argument('--mode', help='Use production (prod) or development (dev) section of config', choices=['prod', 'dev'], required=True) parser.add_argument('--config', default='config.cfg', help='Specify non standard config file name') parser.add_argument('--debug', help='Run Flask in debug mode', action='store_true') args = vars(parser.parse_args()) config = Config.load(args.get('config'), args.get('mode')) database_auth = config.get('database_auth') Database.set_database_name('rereco') if database_auth: Database.set_credentials_file(database_auth) Database.add_search_rename('requests', 'runs', 'runs<int>') Database.add_search_rename('requests', 'run', 'runs<int>') Database.add_search_rename('requests', 'workflows', 'workflows.name') Database.add_search_rename('requests', 'workflow', 'workflows.name') Database.add_search_rename('requests', 'output_dataset', 'output_datasets') Database.add_search_rename('requests', 'input_dataset', 'input.dataset') Database.add_search_rename('requests', 'input_request', 'input.request') Database.add_search_rename('requests', 'created_on', 'history.0.time') Database.add_search_rename('requests', 'created_by', 'history.0.user') Database.add_search_rename('subcampaigns', 'created_on', 'history.0.time') Database.add_search_rename('subcampaigns', 'created_by', 'history.0.user') Database.add_search_rename('tickets', 'created_on', 'history.0.time') Database.add_search_rename('tickets', 'created_by', 'history.0.user') Database.add_search_rename('tickets', 'subcampaign', 'steps.subcampaign') Database.add_search_rename('tickets', 'processing_string', 'steps.processing_string') debug = args.get('debug', False) port = int(config.get('port', 8002)) host = config.get('host', '0.0.0.0') logger = setup_logging(debug) if os.environ.get('WERKZEUG_RUN_MAIN') != 'true': # Do only once, before the reloader pid = os.getpid() logger.info('PID: %s', pid) with open('rereco.pid', 'w') as pid_file: pid_file.write(str(pid)) logger.info('Starting... Debug: %s, Host: %s, Port: %s', debug, host, port) app.run(host=host, port=port, threaded=True, debug=debug)
""" Script to add run list to relval steps """ import sys import os.path import os sys.path.append(os.path.abspath(os.path.pardir)) from core_lib.database.database import Database Database.set_credentials_file(os.getenv('DB_AUTH')) Database.set_database_name('relval') relvals_database = Database('relvals') total_relvals = relvals_database.get_count() print('Total relvals: %s' % (total_relvals)) for index, item in enumerate(relvals_database.query(limit=total_relvals)): print('Processing entry %s/%s %s' % (index + 1, total_relvals, item.get('prepid', '<no-id>'))) for step in item['steps']: step['input']['run'] = step['input'].get('run', []) relvals_database.save(item) print('Done')
""" Script to update data format in database for PhaseII update """ import sys import os.path import os sys.path.append(os.path.abspath(os.path.pardir)) from core_lib.database.database import Database Database.set_credentials_file(os.getenv('DB_AUTH')) Database.set_database_name('rereco') request_db = Database('requests') subcampaign_db = Database('subcampaigns') old_ticket_db = Database('subcampaign_tickets') new_ticket_db = Database('tickets') total_subcampaigns = subcampaign_db.get_count() total_requests = request_db.get_count() total_old_tickets = old_ticket_db.get_count() total_new_tickets = new_ticket_db.get_count() print('Requests: %s' % (total_requests)) print('Subcampaigns: %s' % (total_subcampaigns)) print('(Old) subcampaign tickets: %s' % (total_old_tickets)) print('(New) tickets: %s' % (total_new_tickets)) for index, subcampaign in enumerate( subcampaign_db.query(limit=total_subcampaigns)): print('Processing subcampaign %s/%s %s' % (index + 1, total_subcampaigns, subcampaign['prepid']))