def move_to_done(database_auth, port):
"""
Try to move all submitted RelVals to next status
"""
Database.set_database_name('relval')
Database.set_credentials_file(database_auth)
connection = http.client.HTTPConnection('localhost',
port=port,
timeout=300)
headers = {
'Content-Type': 'application/json',
'Adfs-Login': '******',
'Adfs-Group': 'cms-pdmv-serv'
}
relval_db = Database('relvals')
relvals = [{}]
page = 0
while relvals:
relvals = relval_db.query(query_string='status=submitted', page=page)
page += 1
for relval in relvals:
print(relval['prepid'])
connection.request('POST',
'/api/relvals/next_status',
json.dumps(relval),
headers=headers)
response = connection.getresponse()
response_text = json.loads(response.read())['message']
print(' %s %s' % (response.code, response_text))
def dump_documents(database_auth, output_dir, database_name, collections):
"""
Dump a list of collections to separate directories
"""
Database.set_database_name(database_name)
Database.set_credentials_file(database_auth)
for collection_name in collections:
print('Collection %s' % (collection_name))
database = Database(collection_name)
collection = database.collection
doc_count = collection.count_documents({})
print('Found %s documents' % (doc_count))
documents = [{}]
page = 0
limit = 100
collection_path = f'{output_dir}/{collection_name}'
os.makedirs(collection_path)
while documents:
documents = collection.find({}).sort('_id', 1).skip(page * limit).limit(limit)
documents = [d for d in documents]
if not documents:
break
file_name = f'{collection_path}/{database_name}_{collection_name}_{page}.json'
with open(file_name, 'w') as output_file:
json.dump(documents, output_file)
print('Page %s done' % (page))
page += 1
def main():
"""
Main function: start Flask web server
"""
parser = argparse.ArgumentParser(description='ReReco Machine')
parser.add_argument('--mode',
help='Use production (prod) or development (dev) section of config',
choices=['prod', 'dev'],
required=True)
parser.add_argument('--config',
default='config.cfg',
help='Specify non standard config file name')
parser.add_argument('--debug',
help='Run Flask in debug mode',
action='store_true')
args = vars(parser.parse_args())
config = Config.load(args.get('config'), args.get('mode'))
database_auth = config.get('database_auth')
Database.set_database_name('rereco')
if database_auth:
Database.set_credentials_file(database_auth)
Database.add_search_rename('requests', 'runs', 'runs<int>')
Database.add_search_rename('requests', 'run', 'runs<int>')
Database.add_search_rename('requests', 'workflows', 'workflows.name')
Database.add_search_rename('requests', 'workflow', 'workflows.name')
Database.add_search_rename('requests', 'output_dataset', 'output_datasets')
Database.add_search_rename('requests', 'input_dataset', 'input.dataset')
Database.add_search_rename('requests', 'input_request', 'input.request')
Database.add_search_rename('requests', 'created_on', 'history.0.time')
Database.add_search_rename('requests', 'created_by', 'history.0.user')
Database.add_search_rename('subcampaigns', 'created_on', 'history.0.time')
Database.add_search_rename('subcampaigns', 'created_by', 'history.0.user')
Database.add_search_rename('tickets', 'created_on', 'history.0.time')
Database.add_search_rename('tickets', 'created_by', 'history.0.user')
Database.add_search_rename('tickets', 'subcampaign', 'steps.subcampaign')
Database.add_search_rename('tickets', 'processing_string', 'steps.processing_string')
debug = args.get('debug', False)
port = int(config.get('port', 8002))
host = config.get('host', '0.0.0.0')
logger = setup_logging(debug)
if os.environ.get('WERKZEUG_RUN_MAIN') != 'true':
# Do only once, before the reloader
pid = os.getpid()
logger.info('PID: %s', pid)
with open('rereco.pid', 'w') as pid_file:
pid_file.write(str(pid))
logger.info('Starting... Debug: %s, Host: %s, Port: %s', debug, host, port)
app.run(host=host,
port=port,
threaded=True,
debug=debug)
"""
Script to add run list to relval steps
"""
import sys
import os.path
import os
sys.path.append(os.path.abspath(os.path.pardir))
from core_lib.database.database import Database
Database.set_credentials_file(os.getenv('DB_AUTH'))
Database.set_database_name('relval')
relvals_database = Database('relvals')
total_relvals = relvals_database.get_count()
print('Total relvals: %s' % (total_relvals))
for index, item in enumerate(relvals_database.query(limit=total_relvals)):
print('Processing entry %s/%s %s' %
(index + 1, total_relvals, item.get('prepid', '<no-id>')))
for step in item['steps']:
step['input']['run'] = step['input'].get('run', [])
relvals_database.save(item)
print('Done')
