def place_replica(once=False, thread=0, did_queue=None, waiting_time=100, dry_run=False, sampling=False, algorithms='t2_free_space_only_pop_with_network', datatypes='NTUP,DAOD', dest_rse_expr='type=DATADISK', max_bytes_hour=100000000000000, max_files_hour=100000, max_bytes_hour_rse=50000000000000, max_files_hour_rse=10000, min_popularity=8, min_recent_requests=5, max_replicas=5, sleep_time=10): """ Thread to run the placement algorithm to decide if and where to put new replicas. """ try: c3po_options = config_get_options('c3po') client = None if 'algorithms' in c3po_options: algorithms = config_get('c3po', 'algorithms') algorithms = algorithms.split(',') if not dry_run: if len(algorithms) != 1: logging.error('Multiple algorithms are only allowed in dry_run mode') return client = Client(auth_type='x509_proxy', account='c3po', creds={'client_proxy': '/opt/rucio/etc/ddmadmin.long.proxy'}) vo = client.vo instances = {} for algorithm in algorithms: module_path = 'rucio.daemons.c3po.algorithms.' + algorithm module = __import__(module_path, globals(), locals(), ['PlacementAlgorithm']) instance = module.PlacementAlgorithm(datatypes, dest_rse_expr, max_bytes_hour, max_files_hour, max_bytes_hour_rse, max_files_hour_rse, min_popularity, min_recent_requests, max_replicas) instances[algorithm] = instance params = { 'dry_run': dry_run, 'sampling': sampling, 'datatypes': datatypes, 'dest_rse_expr': dest_rse_expr, 'max_bytes_hour': max_bytes_hour, 'max_files_hour': max_files_hour, 'max_bytes_hour_rse': max_bytes_hour_rse, 'max_files_hour_rse': max_files_hour_rse, 'min_recent_requests': min_recent_requests, 'min_popularity': min_popularity } instance_id = str(uuid4()).split('-')[0] elastic_url = config_get('c3po', 'elastic_url') elastic_index = config_get('c3po', 'elastic_index') ca_cert = False if 'ca_cert' in c3po_options: ca_cert = config_get('c3po', 'ca_cert') auth = False if ('elastic_user' in c3po_options) and ('elastic_pass' in c3po_options): auth = HTTPBasicAuth(config_get('c3po', 'elastic_user'), config_get('c3po', 'elastic_pass')) w = waiting_time while not GRACEFUL_STOP.is_set(): if w < waiting_time: w += sleep_time sleep(sleep_time) continue len_dids = did_queue.qsize() if len_dids > 0: logging.debug('(%s) %d did(s) in queue' % (instance_id, len_dids)) else: logging.debug('(%s) no dids in queue' % (instance_id)) for _ in range(0, len_dids): did = did_queue.get() if isinstance(did[0], string_types): did[0] = InternalScope(did[0], vo=vo) for algorithm, instance in instances.items(): logging.info('(%s:%s) Retrieved %s:%s from queue. Run placement algorithm' % (algorithm, instance_id, did[0], did[1])) decision = instance.place(did) decision['@timestamp'] = datetime.utcnow().isoformat() decision['algorithm'] = algorithm decision['instance_id'] = instance_id decision['params'] = params create_rule = True if sampling and 'error_reason' not in decision: create_rule = bool(ord(md5(decision['did']).hexdigest()[-1]) & 1) decision['create_rule'] = create_rule # write the output to ES for further analysis index_url = elastic_url + '/' + elastic_index + '-' + datetime.utcnow().strftime('%Y-%m') + '/record/' try: if ca_cert: r = post(index_url, data=dumps(decision), verify=ca_cert, auth=auth) else: r = post(index_url, data=dumps(decision)) if r.status_code != 201: logging.error(r) logging.error('(%s:%s) could not write to ElasticSearch' % (algorithm, instance_id)) except RequestException as e: logging.error('(%s:%s) could not write to ElasticSearch' % (algorithm, instance_id)) logging.error(e) continue logging.debug(decision) if 'error_reason' in decision: logging.error('(%s:%s) The placement algorithm ran into an error: %s' % (algorithm, instance_id, decision['error_reason'])) continue logging.info('(%s:%s) Decided to place a new replica for %s on %s' % (algorithm, instance_id, decision['did'], decision['destination_rse'])) if (not dry_run) and create_rule: # DO IT! try: add_rule(client, {'scope': did[0].external, 'name': did[1]}, decision.get('source_rse'), decision.get('destination_rse')) except exception.RucioException as e: logging.debug(e) w = 0 except Exception as e: logging.critical(e)
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from json import dumps, loads from requests import post from requests.auth import HTTPBasicAuth from rucio.common.config import config_get, config_get_options ELASTIC_URL = config_get('es-atlas', 'url') ELASTIC_OPTIONS = config_get_options('es-atlas') AUTH = None if ('username' in ELASTIC_OPTIONS) and ('password' in ELASTIC_OPTIONS): AUTH = HTTPBasicAuth(config_get('es-atlas', 'username'), config_get('es-atlas', 'password')) if 'ca_cert' in ELASTIC_OPTIONS: ELASTIC_CA_CERT = config_get('es-atlas', 'ca_cert') else: ELASTIC_CA_CERT = False URL = ELASTIC_URL + '/atlas_rucio-popularity-*/_search' def get_popularity(did): """