def _child(self, pipe, args, kwargs): signal.alarm(30) if not isinstance(self.cred, basestring): os.environ['X509_USER_CERT'] = self.cred.name os.environ['X509_USER_KEY'] = self.cred.name os.environ['X509_USER_PROXY'] = self.cred.name exit_code = 0 try: if context_type is None: raise RuntimeError('Could not load the gfal2 python module') ctx = context_type() if isinstance(self.cred, basestring): # A IAM token is used for authentication, set it in the context s_cred = gfal2.cred_new("BEARER", self.cred.split(':')[0]) try: if isinstance(args[0], dict): if "surl" in args[0].keys(): domain = urlparse.urlparse( args[0]["surl"]).hostname else: domain = urlparse.urlparse(args[0]["old"]).hostname else: domain = urlparse.urlparse(args[0]).hostname except Exception, e: # Will get a 401 from storage domain = "" gfal2.cred_set(ctx, domain, s_cred) result = self.method(ctx, *args, **kwargs) pipe.write(json.dumps(result))
def parse_args(args, io): if args.srmv in svcVers: svcVer = str(args.srmv) else: errstr = 'srmv parameter must be one of '+ \ ', '.join([x for x in svcVers])+'. '+args.srmv+' given' io.set_status(nap.CRITICAL, errstr) return 1 os.environ['LCG_GFAL_INFOSYS'] = args.ldap_url if args.x509: cred = gfal2.cred_new("X509_CERT", args.x509) gfal2.cred_set(ctx, "srm://", cred) gfal2.cred_set(ctx, "gsiftp://", cred)
def pdm_gfal_copy( copy_dict, s_cred_file=None, t_cred_file=None, overwrite=False, # pylint: disable=too-many-arguments, too-many-locals parent=True, nbstreams=1, timeout=None, verbosity=logging.INFO): """ Copy a single source file to a target file. Use separate source and target credentials. Do not overwrite destination by default. Copy json string is of a form: '{"files":[(source1, dest1), (source2, dest2),...]}' """ # _logger.addHandler(logging.StreamHandler()) _logger.setLevel(verbosity) copy_list = copy_dict.get('files', []) if not copy_list: _logger.warning("No files to copy") dump_and_flush({ "Reason": "No files to copy passed in", "Code": 1, 'id': '' }) return if _logger.isEnabledFor(logging.DEBUG): for job_id, f_source, f_dest in copy_list: _logger.debug( "job id %s : gfal copy source: %s TO dest: %s , overwrite ? %s ", job_id, f_source, f_dest, overwrite) s_cred = _get_cred(s_cred_file) t_cred = _get_cred(t_cred_file) if s_cred is None or t_cred is None: _logger.fatal( "Please provide credential location: source %s, dest %s. ", s_cred, t_cred) dump_and_flush({ "Reason": "No credentials passed in", "Code": 1, 'id': '' }) return ctx = gfal2.creat_context() params = ctx.transfer_parameters() params.overwrite = overwrite params.create_parent = parent params.nbstreams = nbstreams if timeout is not None: params.timeout = timeout # unzip: _, src_l, dst_l = zip(*copy_list) # don't care about jobid s_root = str(os.path.dirname(os.path.commonprefix(src_l))) d_root = str(os.path.dirname(os.path.commonprefix(dst_l))) _logger.info("common source prefix: %s ", s_root) _logger.info("common dest prefix: %s ", d_root) gfal2.cred_set(ctx, s_root, s_cred) gfal2.cred_set(ctx, d_root, t_cred) for jobid, source_file, dest_file in copy_list: try: job = {'jobid': jobid, 'monitor': False} params.event_callback = partial(event_callback, job) params.monitor_callback = partial(monitor_callback, job) start_time = time.time() res = ctx.filecopy(params, str(source_file), str(dest_file)) if not job['monitor']: # pseudo-monitoring elapsed = time.time() - start_time dump_and_flush({ 'id': jobid, 'transferred': -1, 'elapsed': elapsed, 'average': -1, 'instant': -1 }) dump_and_flush({'Code': res, 'Reason': 'OK', 'id': jobid}) except gfal2.GError as gerror: dump_and_flush({ 'Code': 1, 'Reason': str(gerror), 'id': jobid }, _logger, str(gerror)) return
if action not in [ "list-classes", "list-transitions", "get-qos", "get-target-qos", "change-qos" ]: parser.error("Unrecognized action: %s" % action) if action == "change-qos": if len(args) < 4: parser.error("Missing <target-qos> argument!") target_qos = args[3] # Create Gfal2 handler context = gfal2.creat_context() cred = gfal2.cred_new("BEARER", token) host = '{uri.scheme}://{uri.netloc}/'.format(uri=urlparse(url)) gfal2.cred_set(context, host, cred) if action == "list-classes": print "Dataobject: %s" % context.qos_check_classes(host, "dataobject") print "Container: %s" % context.qos_check_classes(host, "container") elif action == "list-transitions": print context.check_available_qos_transitions(url) elif action == "get-qos": print context.check_file_qos(url) elif action == "get-target-qos": print context.check_target_qos(url) elif action == "change-qos": print context.change_object_qos(url, target_qos) else: # Should not reach here print "Unrecognized action"
if options.checksum: (alg, val) = options.checksum.split(':') params.set_user_defined_checksum(alg, val) print("User defined checksum: %s:%s" % params.get_user_defined_checksum()) if options.source_space_token: params.src_spacetoken = options.source_space_token print("Source space token: %s" % params.src_spacetoken) if options.dest_space_token: params.dst_spacetoken = options.dest_space_token print("Destination space token: %s" % params.dst_spacetoken) if options.source_cred: s_cred = gfal2.cred_new("X509_CERT", options.source_cred) gfal2.cred_set(ctx, source, s_cred) print("Source credentials: %s" % options.source_cred) if options.dest_cred: d_cred = gfal2.cred_new("X509_CERT", options.dest_cred) gfal2.cred_set(ctx, dest, d_cred) print("Destination credentials: %s" % options.dest_cred) # Five minutes timeout params.timeout = 300 # Copy! # Keep in mind source and destination can have different protocols, # and gfal2 will deal with it # (i.e. source can be file:/// and destination gsiftp://) try: r = ctx.filecopy(params, source, dest) print("Copy succeeded!")
import logging streamHandler = logging.StreamHandler(sys.stdout) streamHandler.setFormatter( logging.Formatter( "%(asctime)s.%(msecs)03d [%(levelname)s](%(module)s:%(lineno)d) %(message)s", "%d %b %H:%M:%S")) gfal2_logger = logging.getLogger("gfal2") gfal2_logger.addHandler(streamHandler) gfal2_logger.setLevel(logging.DEBUG) gfal2.set_verbose(gfal2.verbose_level.debug) gfal2.set_verbose(gfal2.verbose_level.trace) dst = 'davs://storm.example:8443/wlcg/test-not-found' ctx = gfal2.creat_context() ctx.set_opt_boolean("HTTP PLUGIN", "KEEP_ALIVE", False) cred = gfal2.cred_new("BEARER", os.getenv('BEARER_TOKEN')) gfal2.cred_set(ctx, dst, cred) try: ctx.stat(dst) except: pass try: ctx.stat(dst) except: pass