def discover_blocks(options): # Get work directory, create dbs dump directory if os.path.isdir(options.args[0]): work_dn = os.path.abspath(os.path.normpath(options.args[0])) else: work_dn = gc_create_config(config_file=options.args[0]).get_work_path() if not options.opts.tempdir: options.opts.tempdir = os.path.join(work_dn, 'dbs') if not os.path.exists(options.opts.tempdir): os.mkdir(options.opts.tempdir) # get provider with dataset information config = gc_create_config(config_dict={'dataset': options.config_dict}, load_old_config=False) if options.opts.input_file: provider = DataProvider.create_instance('ListProvider', config, 'dataset', options.opts.input_file) else: provider = DataProvider.create_instance('DBSInfoProvider', config, 'dataset', options.args[0]) blocks = provider.get_block_list_cached(show_stats=False) DataProvider.save_to_file(os.path.join(options.opts.tempdir, 'dbs.dat'), blocks) if options.opts.discovery: sys.exit(os.EX_OK) return blocks
def discover_blocks(options): # Get work directory, create dbs dump directory if os.path.isdir(options.args[0]): work_dn = os.path.abspath(os.path.normpath(options.args[0])) else: work_dn = gc_create_config(config_file=options.args[0]).get_work_path() if not options.opts.tempdir: options.opts.tempdir = os.path.join(work_dn, 'dbs') if not os.path.exists(options.opts.tempdir): os.mkdir(options.opts.tempdir) # get provider with dataset information config = gc_create_config(config_dict={'dataset': options.config_dict}, load_old_config=False) if options.opts.input_file: provider = DataProvider.create_instance('ListProvider', config, 'dataset', options.opts.input_file) else: provider = DataProvider.create_instance('DBSInfoProvider', config, 'dataset', options.args[0]) blocks = provider.get_block_list_cached(show_stats=False) DataProvider.save_to_file(os.path.join(options.opts.tempdir, 'dbs.dat'), blocks) if options.opts.discovery: sys.exit(os.EX_OK) return blocks
def setup_config(opts, args): # Set config based on settings from config file or command line config_fn = None if os.path.exists(args[0]): config_fn = args[0] config = gc_create_config(config_file=config_fn).change_view( set_sections=['global']) if os.path.exists(config.get_work_path('datamap.tar')): opts.dataset = config.get_work_path('datamap.tar') config.change_view(set_sections=['jobs']).set('nseeds', '1', '?=') param_config = config.change_view(set_sections=['parameters']) if opts.parameter: logging.info('Provided options:') for param in opts.parameter: key, value = param.split('=', 1) param_config.set(key.strip(), value.strip().replace('\\n', '\n'), '=') logging.info('\t%s: %s', key.strip(), value.strip()) logging.info('') if config_fn is None: param_config.set('parameters', str.join(' ', args).replace('\\n', '\n')) if opts.dataset: param_config.set('default lookup', 'DATASETNICK') if opts.verbose > 2: buffer = StringBuffer() config.change_view(set_sections=None).write(buffer) logging.getLogger('script').info(buffer.getvalue().rstrip()) return config
def setup_config(opts, args): # Set config based on settings from config file or command line config_fn = None if os.path.exists(args[0]): config_fn = args[0] config = gc_create_config(config_file=config_fn).change_view(set_sections=['global']) if os.path.exists(config.get_work_path('datamap.tar')): opts.dataset = config.get_work_path('datamap.tar') config.change_view(set_sections=['jobs']).set('nseeds', '1', '?=') param_config = config.change_view(set_sections=['parameters']) if opts.parameter: logging.info('Provided options:') for param in opts.parameter: key, value = param.split('=', 1) param_config.set(key.strip(), value.strip().replace('\\n', '\n'), '=') logging.info('\t%s: %s', key.strip(), value.strip()) logging.info('') if config_fn is None: param_config.set('parameters', str.join(' ', args).replace('\\n', '\n')) if opts.dataset: param_config.set('default lookup', 'DATASETNICK') if opts.verbose > 2: buffer = StringBuffer() config.change_view(set_sections=None).write(buffer) logging.getLogger('script').info(buffer.getvalue().rstrip()) return config
def get_job_db(options): if len(options.args) != 1: options.parser.exit_with_usage(options.parser.usage('jobs')) config = gc_create_config(config_file=options.args[0]) # Initialise task module task_cls_name = config.get(['task', 'module']) task = Plugin.create_instance(task_cls_name, config, task_cls_name) job_db = Plugin.create_instance('TextFileJobDB', config) selected = JobSelector.create(options.opts.job_selector, task=task) logging.info('Matching jobs: ' + str.join(' ', imap(str, job_db.iter_jobs(selected))))
def get_job_db(options): if len(options.args) != 1: options.parser.exit_with_usage(options.parser.usage('jobs')) config = gc_create_config(config_file=options.args[0]) # Initialise task module task_cls_name = config.get(['task', 'module']) task = Plugin.create_instance(task_cls_name, config, task_cls_name) job_db = Plugin.create_instance('TextFileJobDB', config) selected = JobSelector.create(options.opts.job_selector, task=task) logging.info('Matching jobs: ' + str.join(' ', imap(str, job_db.iter_jobs(selected))))
def _main(): parser = ScriptOptions(usage='%s [OPTIONS] <DBS dataset path>') parser.add_text(None, '', 'producer', default='SimpleNickNameProducer', help='Name of the nickname producer') parser.add_bool(None, 'L', 'nick-list', default=False, help='List available nickname producer classes') options = parser.script_parse() if options.opts.nick_list: display_plugin_list_for('NickNameProducer', title='Available nickname producer classes') if not options.args: parser.exit_with_usage() dataset_path = options.args[0] if ('*' in dataset_path) or os.path.exists(dataset_path): dataset_provider = 'DBS3Provider' if os.path.exists(dataset_path): dataset_provider = 'ListProvider' provider = Plugin.create_instance(dataset_provider, gc_create_config(), 'dataset', dataset_path) dataset_path_list = provider.get_dataset_name_list() else: dataset_path_list = [dataset_path] nn_prod = Plugin.get_class('NickNameProducer').create_instance( options.opts.producer, gc_create_config(), 'dataset') ConsoleTable.create([(1, 'Dataset'), (0, 'Nickname')], lmap( lambda ds: { 0: nn_prod.get_name('', ds, None), 1: ds }, dataset_path_list), 'll')
def get_dataset_config(opts, args): dataset = str.join('\n', args) if (not opts.provider) and not os.path.exists(clean_path(dataset.split()[0])): opts.provider = 'DBS3Provider' elif not opts.provider: opts.provider = 'ListProvider' config_dict = {'dbs blacklist T1 *': 'False', 'remove empty blocks *': 'False', 'remove empty files *': 'False', 'location format *': opts.location, 'nickname check collision *': 'False', 'dataset location filter': opts.location_filter, 'dataset *': dataset, 'dataset provider *': opts.provider} if opts.metadata or opts.list_metadata or opts.list_metadata_common: config_dict['lumi filter *'] = '-' config_dict['keep lumi metadata *'] = 'True' return gc_create_config(config_file=opts.settings, config_dict={'dataset': config_dict})
def backend_list(finder_name): finder = Plugin.get_class('BackendDiscovery').create_instance(finder_name, gc_create_config()) item_dict_list = [] item_key_set = set() for item_dict in finder.discover(): nice_item_dict = {} for (key, value) in item_dict.items(): if isinstance(key, int): key = WMS.enum2str(key) nice_item_dict[key] = value item_key_set.add(key) item_dict_list.append(nice_item_dict) item_key_set.remove('name') item_key_list = sorted(item_key_set) ConsoleTable.create([('name', 'Name')] + lzip(item_key_list, item_key_list), item_dict_list)
def _lookup_pfn(options): if '<hypernews_name>' in options.opts.lfn: token = Plugin.get_class('AccessToken').create_instance('VomsProxy', gc_create_config(), 'token') site_db = SiteDB() hn_name = site_db.dn_to_username(dn=token.get_fq_user_name()) if not hn_name: raise Exception('Unable to map grid certificate to hypernews name!') options.opts.lfn = options.opts.lfn.replace('<hypernews_name>', hn_name) tmp = _lfn2pfn(node=options.opts.se, prot=options.opts.se_prot, lfn=options.opts.lfn) for entry in tmp: entry_str = entry['pfn'] if len(tmp) > 1: # write node name if multiple mappings are returned entry_str = '%s %s' % (entry['node'], entry_str) logging.getLogger('script').info(entry_str)
def discover_dataset(provider_name, config_dict): buffer = StringBuffer() config = gc_create_config(config_dict={'dataset': config_dict}) config = config.change_view(set_sections=['dataset']) provider = Plugin.get_class('DataProvider').create_instance(provider_name, config, 'dataset', config_dict['dataset'], None) if config_dict['dump config'] == 'True': config.write(buffer, print_default=True, print_minimal=True) return logging.getLogger('script').info(buffer.getvalue().rstrip()) strip_metadata = config_dict['strip'] == 'True' block_iter = provider.get_block_list_cached(show_stats=False) if config_dict['output']: return provider.save_to_file(config_dict['output'], block_iter, strip_metadata) for _ in provider.save_to_stream(buffer, block_iter, strip_metadata): pass logging.getLogger('script').info(buffer.getvalue().rstrip())
def discover_dataset(provider_name, config_dict): buffer = StringBuffer() config = gc_create_config(config_dict={'dataset': config_dict}) config = config.change_view(set_sections=['dataset']) provider = Plugin.get_class('DataProvider').create_instance( provider_name, config, 'dataset', config_dict['dataset'], None) if config_dict['dump config'] == 'True': config.write(buffer, print_default=True, print_minimal=True) return logging.getLogger('script').info(buffer.getvalue().rstrip()) strip_metadata = config_dict['strip'] == 'True' block_iter = provider.get_block_list_cached(show_stats=False) if config_dict['output']: return provider.save_to_file(config_dict['output'], block_iter, strip_metadata) for _ in provider.save_to_stream(buffer, block_iter, strip_metadata): pass logging.getLogger('script').info(buffer.getvalue().rstrip())
def backend_list(finder_name): finder = Plugin.get_class('BackendDiscovery').create_instance( finder_name, gc_create_config()) item_dict_list = [] item_key_set = set() for item_dict in finder.discover(): nice_item_dict = {} for (key, value) in item_dict.items(): if isinstance(key, int): key = WMS.enum2str(key) nice_item_dict[key] = value item_key_set.add(key) item_dict_list.append(nice_item_dict) item_key_set.remove('name') item_key_list = sorted(item_key_set) ConsoleTable.create([('name', 'Name')] + lzip(item_key_list, item_key_list), item_dict_list)
def get_dataset_config(opts, args): dataset = str.join('\n', args) if (not opts.provider) and not os.path.exists( clean_path(dataset.split()[0])): opts.provider = 'DBS3Provider' elif not opts.provider: opts.provider = 'ListProvider' config_dict = { 'dbs blacklist T1 *': 'False', 'remove empty blocks *': 'False', 'remove empty files *': 'False', 'location format *': opts.location, 'nickname check collision *': 'False', 'dataset location filter': opts.location_filter, 'dataset *': dataset, 'dataset provider *': opts.provider } if opts.metadata or opts.list_metadata or opts.list_metadata_common: config_dict['lumi filter *'] = '-' config_dict['keep lumi metadata *'] = 'True' return gc_create_config(config_file=opts.settings, config_dict={'dataset': config_dict})