def _enter_title(ui, dataset): default = _get_default_title(dataset) while True: title = ui.question( "Please enter the title (must be at least 3 characters long).", title="New article", default=default) if len(title) < 3: ui.error("Title must be at least 3 characters long.") else: return title
def _run_with_exception_handler(cmdlineargs): """Execute the command and perform some reporting normalization if it crashes, but otherwise just let it go""" # otherwise - guard and only log the summary. Postmortem is not # as convenient if being caught in this ultimate except try: return cmdlineargs.func(cmdlineargs) # catch BaseException for KeyboardInterrupt except BaseException as exc: from datalad.support.exceptions import ( CapturedException, InsufficientArgumentsError, IncompleteResultsError, CommandError, ) ce = CapturedException(exc) # we crashed, it has got to be non-zero for starters exit_code = 1 if isinstance(exc, InsufficientArgumentsError): # if the func reports inappropriate usage, give help output lgr.error('%s (%s)', ce, exc.__class__.__name__) cmdlineargs.subparser.print_usage(sys.stderr) exit_code = 2 elif isinstance(exc, IncompleteResultsError): # in general we do not want to see the error again, but # present in debug output lgr.debug('could not perform all requested actions: %s', ce) elif isinstance(exc, CommandError): exit_code = _communicate_commanderror(exc) or exit_code elif isinstance(exc, KeyboardInterrupt): from datalad.ui import ui ui.error("\nInterrupted by user while doing magic: %s" % ce) exit_code = 3 else: # some unforeseen problem lgr.error('%s (%s)', ce.message, ce.name) sys.exit(exit_code)
def enter_new(self, url=None, auth_types=[], url_re=None, name=None, credential_name=None, credential_type=None): # TODO: level/location! """Create new provider and credential config If interactive, this will ask the user to enter the details (or confirm default choices). A dedicated config file is written at <user_config_dir>/providers/<name>.cfg Parameters: ----------- url: str or RI URL this config is created for auth_types: list List of authentication types to choose from. First entry becomes default. See datalad.downloaders.providers.AUTHENTICATION_TYPES url_re: str regular expression; Once created, this config will be used for any matching URL; defaults to `url` name: str name for the provider; needs to be unique per user credential_name: str name for the credential; defaults to the provider's name credential_type: str credential type to use (key for datalad.downloaders.CREDENTIAL_TYPES) """ from datalad.ui import ui if url and not name: ri = RI(url) for f in ('hostname', 'name'): try: # might need sanitarization name = str(getattr(ri, f)) except AttributeError: pass known_providers_by_name = {p.name: p for p in self._providers} providers_user_dir = self._get_providers_dirs()['user'] while True: name = ui.question( title="New provider name", text="Unique name to identify 'provider' for %s" % url, default=name) filename = pathjoin(providers_user_dir, '%s.cfg' % name) if name in known_providers_by_name: if ui.yesno( title="Known provider %s" % name, text= "Provider with name %s already known. Do you want to " "use it for this session?" % name, default=True): return known_providers_by_name[name] elif path.lexists(filename): ui.error("File %s already exists, choose another name" % filename) else: break if not credential_name: credential_name = name if not url_re: url_re = re.escape(url) if url else None while True: url_re = ui.question( title="New provider regular expression", text="A (Python) regular expression to specify for which URLs " "this provider should be used", default=url_re) if not re.match(url_re, url): ui.error("Provided regular expression doesn't match original " "url. Please re-enter") # TODO: url_re of another provider might match it as well # I am not sure if we have any kind of "priority" setting ATM # to differentiate or to to try multiple types :-/ else: break authentication_type = None if auth_types: auth_types = [t for t in auth_types if t in AUTHENTICATION_TYPES] if auth_types: authentication_type = auth_types[0] # Setup credential authentication_type = ui.question( title="Authentication type", text="What authentication type to use", default=authentication_type, choices=sorted(AUTHENTICATION_TYPES)) authenticator_class = AUTHENTICATION_TYPES[authentication_type] # TODO: need to figure out what fields that authenticator might # need to have setup and ask for them here! credential_type = ui.question( title="Credential", text="What type of credential should be used?", choices=sorted(CREDENTIAL_TYPES), default=credential_type or getattr(authenticator_class, 'DEFAULT_CREDENTIAL_TYPE')) cfg = self._CONFIG_TEMPLATE.format(**locals()) if ui.yesno(title="Save provider configuration file", text="Following configuration will be written to %s:\n%s" % (filename, cfg), default='yes'): # Just create a configuration file and reload the thing return self._store_new(url=url, authentication_type=authentication_type, authenticator_class=authenticator_class, url_re=url_re, name=name, credential_name=credential_name, credential_type=credential_type, level='user') else: return None
def enter_new(self, url=None, auth_types=[]): from datalad.ui import ui name = None if url: ri = RI(url) for f in ('hostname', 'name'): try: # might need sanitarization name = str(getattr(ri, f)) except AttributeError: pass known_providers_by_name = {p.name: p for p in self._providers} providers_user_dir = self._get_providers_dirs()['user'] while True: name = ui.question( title="New provider name", text="Unique name to identify 'provider' for %s" % url, default=name ) filename = pathjoin(providers_user_dir, '%s.cfg' % name) if name in known_providers_by_name: if ui.yesno( title="Known provider %s" % name, text="Provider with name %s already known. Do you want to " "use it for this session?" % name, default=True ): return known_providers_by_name[name] elif path.lexists(filename): ui.error( "File %s already exists, choose another name" % filename) else: break url_re = re.escape(url) if url else None while True: url_re = ui.question( title="New provider regular expression", text="A (Python) regular expression to specify for which URLs " "this provider should be used", default=url_re ) if not re.match(url_re, url): ui.error("Provided regular expression doesn't match original " "url. Please re-enter") # TODO: url_re of another provider might match it as well # I am not sure if we have any kind of "priority" setting ATM # to differentiate or to to try multiple types :-/ else: break authentication_type = None if auth_types: auth_types = [ t for t in auth_types if t in AUTHENTICATION_TYPES ] if auth_types: authentication_type = auth_types[0] # Setup credential authentication_type = ui.question( title="Authentication type", text="What authentication type to use", default=authentication_type, choices=sorted(AUTHENTICATION_TYPES) ) authenticator_class = AUTHENTICATION_TYPES[authentication_type] # TODO: need to figure out what fields that authenticator might # need to have setup and ask for them here! credential_type = ui.question( title="Credential", text="What type of credential should be used?", choices=sorted(CREDENTIAL_TYPES), default=getattr(authenticator_class, 'DEFAULT_CREDENTIAL_TYPE') ) # Just create a configuration file and reload the thing if not path.lexists(providers_user_dir): os.makedirs(providers_user_dir) cfg = """\ # Provider configuration file created to initially access # {url} [provider:{name}] url_re = {url_re} authentication_type = {authentication_type} # Note that you might need to specify additional fields specific to the # authenticator. Fow now "look into the docs/source" of {authenticator_class} # {authentication_type}_ credential = {name} [credential:{name}] # If known, specify URL or email to how/where to request credentials # url = ??? type = {credential_type} """.format(**locals()) if ui.yesno( title="Save provider configuration file", text="Following configuration will be written to %s:\n%s" % (filename, cfg), default='yes' ): with open(filename, 'wb') as f: f.write(cfg.encode('utf-8')) else: return None self.reload() # XXX see above note about possibly multiple matches etc return self.get_provider(url)
def enter_new(self, url=None, auth_types=[]): from datalad.ui import ui name = None if url: ri = RI(url) for f in ('hostname', 'name'): try: # might need sanitarization name = str(getattr(ri, f)) except AttributeError: pass known_providers_by_name = {p.name: p for p in self._providers} providers_user_dir = self._get_providers_dirs()['user'] while True: name = ui.question( title="New provider name", text="Unique name to identify 'provider' for %s" % url, default=name ) filename = pathjoin(providers_user_dir, '%s.cfg' % name) if name in known_providers_by_name: if ui.yesno( title="Known provider %s" % name, text="Provider with name %s already known. Do you want to " "use it for this session?" % name, default=True ): return known_providers_by_name[name] elif path.lexists(filename): ui.error( "File %s already exists, choose another name" % filename) else: break url_re = re.escape(url) if url else None while True: url_re = ui.question( title="New provider regular expression", text="A (Python) regular expression to specify for which URLs " "this provider should be used", default=url_re ) if not re.match(url_re, url): ui.error("Provided regular expression doesn't match original " "url. Please re-enter") # TODO: url_re of another provider might match it as well # I am not sure if we have any kind of "priority" setting ATM # to differentiate or to to try multiple types :-/ else: break authentication_type = None if auth_types: auth_types = [ t for t in auth_types if t in AUTHENTICATION_TYPES ] if auth_types: authentication_type = auth_types[0] # Setup credential authentication_type = ui.question( title="Authentication type", text="What authentication type to use", default=authentication_type, choices=sorted(AUTHENTICATION_TYPES) ) authenticator_class = AUTHENTICATION_TYPES[authentication_type] # TODO: need to figure out what fields that authenticator might # need to have setup and ask for them here! credential_type = ui.question( title="Credential", text="What type of credential should be used?", choices=sorted(CREDENTIAL_TYPES), default=getattr(authenticator_class, 'DEFAULT_CREDENTIAL_TYPE') ) # Just create a configuration file and reload the thing if not path.lexists(providers_user_dir): os.makedirs(providers_user_dir) cfg = """\ # Provider configuration file created to initially access # {url} [provider:{name}] url_re = {url_re} authentication_type = {authentication_type} # Note that you might need to specify additional fields specific to the # authenticator. Fow now "look into the docs/source" of {authenticator_class} # {authentication_type}_ credential = {name} [credential:{name}] # If known, specify URL or email to how/where to request credentials # url = ??? type = {credential_type} """.format(**locals()) if ui.yesno( title="Save provider configuration file", text="Following configuration will be written to %s:\n%s" % (filename, cfg), default='yes' ): with open(filename, 'wb') as f: f.write(cfg.encode('utf-8')) else: return None self.reload() # XXX see above note about possibly multiple matches etc return self.get_provider(url)
def _ls_s3(loc, fast=False, recursive=False, all_=False, long_=False, config_file=None, list_content=False): """List S3 bucket content""" if loc.startswith('s3://'): bucket_prefix = loc[5:] else: raise ValueError("passed location should be an s3:// url") import boto from hashlib import md5 from boto.s3.key import Key from boto.s3.prefix import Prefix from boto.s3.connection import OrdinaryCallingFormat from boto.exception import S3ResponseError from configparser import ConfigParser as SafeConfigParser if '/' in bucket_prefix: bucket_name, prefix = bucket_prefix.split('/', 1) else: bucket_name, prefix = bucket_prefix, None if prefix and '?' in prefix: ui.message("We do not care about URL options ATM, they get stripped") prefix = prefix[:prefix.index('?')] ui.message("Connecting to bucket: %s" % bucket_name) if config_file: config = SafeConfigParser() config.read(config_file) access_key = config.get('default', 'access_key') secret_key = config.get('default', 'secret_key') # TODO: remove duplication -- reuse logic within downloaders/s3.py to get connected kwargs = {} if '.' in bucket_name: kwargs['calling_format']=OrdinaryCallingFormat() conn = boto.connect_s3(access_key, secret_key, **kwargs) try: bucket = conn.get_bucket(bucket_name) except S3ResponseError as e: ui.message("E: Cannot access bucket %s by name" % bucket_name) all_buckets = conn.get_all_buckets() all_bucket_names = [b.name for b in all_buckets] ui.message("I: Found following buckets %s" % ', '.join(all_bucket_names)) if bucket_name in all_bucket_names: bucket = all_buckets[all_bucket_names.index(bucket_name)] else: raise RuntimeError("E: no bucket named %s thus exiting" % bucket_name) else: # TODO: expose credentials # We don't need any provider here really but only credentials from datalad.downloaders.providers import Providers providers = Providers.from_config_files() provider = providers.get_provider(loc) if not provider: raise ValueError( "Don't know how to deal with this url %s -- no provider defined for %s. " "Define a new provider (DOCS: TODO) or specify just s3cmd config file instead for now." % loc ) downloader = provider.get_downloader(loc) # should authenticate etc, and when ready we will ask for a bucket ;) bucket = downloader.access(lambda url: downloader.bucket, loc) info = [] for iname, imeth in [ ("Versioning", bucket.get_versioning_status), (" Website", bucket.get_website_endpoint), (" ACL", bucket.get_acl), ]: try: ival = imeth() except Exception as e: ival = str(e).split('\n')[0] info.append(" {iname}: {ival}".format(**locals())) ui.message("Bucket info:\n %s" % '\n '.join(info)) kwargs = {} if recursive else {'delimiter': '/'} ACCESS_METHODS = [ bucket.list_versions, bucket.list ] prefix_all_versions = None got_versioned_list = False for acc in ACCESS_METHODS: try: prefix_all_versions = list(acc(prefix, **kwargs)) got_versioned_list = acc is bucket.list_versions break except Exception as exc: lgr.debug("Failed to access via %s: %s", acc, exc_str(exc)) if not prefix_all_versions: ui.error("No output was provided for prefix %r" % prefix) else: max_length = max((len(e.name) for e in prefix_all_versions)) max_size_length = max((len(str(getattr(e, 'size', 0))) for e in prefix_all_versions)) results = [] for e in prefix_all_versions: results.append(e) if isinstance(e, Prefix): ui.message("%s" % (e.name, ),) continue base_msg = ("%%-%ds %%s" % max_length) % (e.name, e.last_modified) if isinstance(e, Key): if got_versioned_list and not (e.is_latest or all_): lgr.debug( "Skipping Key since not all versions requested: %s", e) # Skip this one continue ui.message(base_msg + " %%%dd" % max_size_length % e.size, cr=' ') # OPT: delayed import from datalad.support.s3 import get_key_url url = get_key_url(e, schema='http') try: _ = urlopen(Request(url)) urlok = "OK" except HTTPError as err: urlok = "E: %s" % err.code try: acl = e.get_acl() except S3ResponseError as exc: acl = exc.code if exc.code in ('AccessDenied',) else str(exc) content = "" if list_content: # IO intensive, make an option finally! try: # _ = e.next()[:5] if we are able to fetch the content kwargs = dict(version_id=e.version_id) if list_content in {'full', 'first10'}: if list_content in 'first10': kwargs['headers'] = {'Range': 'bytes=0-9'} content = repr(e.get_contents_as_string(**kwargs)) elif list_content == 'md5': digest = md5() digest.update(e.get_contents_as_string(**kwargs)) content = digest.hexdigest() else: raise ValueError(list_content) # content = "[S3: OK]" except S3ResponseError as err: content = str(err) finally: content = " " + content ui.message( "ver:%-32s acl:%s %s [%s]%s" % (getattr(e, 'version_id', None), acl, url, urlok, content) if long_ else '' ) else: ui.message(base_msg + " " + str(type(e)).split('.')[-1].rstrip("\"'>")) return results