def check(self, name, min_version=None, msg=""): """Check if an external (optionally of specified min version) present Parameters ---------- name: str Name of the external (typically a Python module) min_version: str or version, optional Minimal version to satisfy msg: str, optional An additional message to include into the exception message Raises ------ MissingExternalDependency if the external is completely missing OutdatedExternalDependency if the external is present but does not satisfy the min_version """ ver_present = self[name] if ver_present is None: raise MissingExternalDependency(name, ver=min_version, msg=msg) elif min_version and ver_present < min_version: raise OutdatedExternalDependency(name, ver=min_version, ver_present=ver_present, msg=msg)
def __call__(self, data): input = data.pop(self._input) if self._pop_input else data[self._input] if not Response: # we have no scrapy raise MissingExternalDependency( "scrapy", msg="It is needed for this type of crawling") if isinstance(input, Response): selector = Selector(response=input) if hasattr(input, 'url') and input.url and ('url' not in data): # take the URL of the response object data = updated(data, {'url': input.url}) else: selector = Selector(text=input) count = 0 for entry, data_ in self._select_and_extract(selector, self.query, data): data_ = updated(data_, {self._output: entry.extract()}) # now get associated xpaths, css, etc for selectors_dict, entry_method in ((self._xpaths, entry.xpath), (self._csss, entry.css)): if not selectors_dict: continue for key in selectors_dict: selector_ = selectors_dict[key] key_extracted = entry_method(selector_).extract() if not len(key_extracted): # TODO: warning, make mandatory to have a hit if expected? continue if len(key_extracted) > 1: if self._allow_multiple: data_[key] = key_extracted # raise NotImplementedError("Don't know what to do yet with this one") else: lgr.warn( "Got multiple selections for xpath query %s. " "Keeping only the first one: %s" % (repr(selector_), key_extracted[0])) data_[key] = key_extracted[0] else: data_[key] = key_extracted[0] count += 1 yield data_ if self._min_count and count < self._min_count: raise ValueError( "Did not match required %d matches (got %d) using %s" % (self._min_count, count, self)) if self._max_count and count > self._max_count: raise ValueError("Matched more than %d matches (got %d) using %s" % (self._max_count, count, self))
def _parse_date(date): if date.startswith("@"): # unix timestamp timestamp = int(date[1:]) else: try: import dateutil.parser except ImportError: raise MissingExternalDependency( "python-dateutil", msg="This package is required to parse non-timestamp dates") from calendar import timegm # Note: datetime.timestamp isn't available in Python 2. try: timestamp = timegm(dateutil.parser.parse(date).utctimetuple()) except TypeError as exc: # Make older dateutil versions return a consistent error for # invalid dates. raise ValueError(exc) return timestamp
def __call__(sshurl, name=None, target_dir=None, target_url=None, target_pushurl=None, dataset=None, recursive=False, recursion_limit=None, existing='error', shared=None, group=None, ui=False, as_common_datasrc=None, publish_by_default=None, publish_depends=None, annex_wanted=None, annex_group=None, annex_groupwanted=None, inherit=False, since=None): # # nothing without a base dataset # ds = require_dataset(dataset, check_installed=True, purpose='creating a sibling') refds_path = ds.path # # all checks that are possible before we start parsing the dataset # # possibly use sshurl to get the name in case if not specified if not sshurl: if not inherit: raise InsufficientArgumentsError( "needs at least an SSH URL, if no inherit option" ) if name is None: raise ValueError( "Neither SSH URL, nor the name of sibling to inherit from " "was specified" ) # It might well be that we already have this remote setup try: sshurl = CreateSibling._get_remote_url(ds, name) except Exception as exc: lgr.debug('%s does not know about url for %s: %s', ds, name, exc_str(exc)) elif inherit: raise ValueError( "For now, for clarity not allowing specifying a custom sshurl " "while inheriting settings" ) # may be could be safely dropped -- still WiP if not sshurl: # TODO: may be more back up before _prep? super_ds = ds.get_superdataset() if not super_ds: raise ValueError( "Could not determine super dataset for %s to inherit URL" % ds ) super_url = CreateSibling._get_remote_url(super_ds, name) # for now assuming hierarchical setup # (TODO: to be able to destinguish between the two, probably # needs storing datalad.*.target_dir to have %RELNAME in there) sshurl = slash_join(super_url, relpath(ds.path, super_ds.path)) # check the login URL sshri = RI(sshurl) if not is_ssh(sshri): raise ValueError( "Unsupported SSH URL: '{0}', " "use ssh://host/path or host:path syntax".format(sshurl)) if not name: # use the hostname as default remote name name = sshri.hostname lgr.debug( "No sibling name given, use URL hostname '%s' as sibling name", name) if since == '': # consider creating siblings only since the point of # the last update # XXX here we assume one to one mapping of names from local branches # to the remote active_branch = ds.repo.get_active_branch() since = '%s/%s' % (name, active_branch) # # parse the base dataset to find all subdatasets that need processing # to_process = [] for ap in AnnotatePaths.__call__( dataset=refds_path, # only a single path! path=refds_path, recursive=recursive, recursion_limit=recursion_limit, action='create_sibling', # both next should not happen anyways unavailable_path_status='impossible', nondataset_path_status='error', modified=since, return_type='generator', on_failure='ignore'): if ap.get('status', None): # this is done yield ap continue if ap.get('type', None) != 'dataset' or ap.get('state', None) == 'absent': # this can happen when there is `since`, but we have no # use for anything but datasets here continue checkds_remotes = Dataset(ap['path']).repo.get_remotes() \ if ap.get('state', None) != 'absent' \ else [] if publish_depends: # make sure dependencies are valid # TODO: inherit -- we might want to automagically create # those dependents as well??? unknown_deps = set(assure_list(publish_depends)).difference(checkds_remotes) if unknown_deps: ap['status'] = 'error' ap['message'] = ( 'unknown sibling(s) specified as publication dependency: %s', unknown_deps) yield ap continue if name in checkds_remotes and existing in ('error', 'skip'): ap['status'] = 'error' if existing == 'error' else 'notneeded' ap['message'] = ( "sibling '%s' already configured (specify alternative name, or force " "reconfiguration via --existing", name) yield ap continue to_process.append(ap) if not to_process: # we ruled out all possibilities # TODO wait for gh-1218 and make better return values lgr.info("No datasets qualify for sibling creation. " "Consider different settings for --existing " "or --since if this is unexpected") return if target_dir is None: if sshri.path: target_dir = sshri.path else: target_dir = '.' # TODO: centralize and generalize template symbol handling replicate_local_structure = "%RELNAME" not in target_dir # request ssh connection: lgr.info("Connecting ...") assert(sshurl is not None) # delayed anal verification ssh = ssh_manager.get_connection(sshurl) if not ssh.get_annex_version(): raise MissingExternalDependency( 'git-annex', msg='on the remote system') # # all checks done and we have a connection, now do something # # loop over all datasets, ordered from top to bottom to make test # below valid (existing directories would cause the machinery to halt) # But we need to run post-update hook in depth-first fashion, so # would only collect first and then run (see gh #790) yielded = set() remote_repos_to_run_hook_for = [] for currentds_ap in \ sorted(to_process, key=lambda x: x['path'].count('/')): current_ds = Dataset(currentds_ap['path']) path = _create_dataset_sibling( name, current_ds, ds.path, ssh, replicate_local_structure, sshri, target_dir, target_url, target_pushurl, existing, shared, group, publish_depends, publish_by_default, ui, as_common_datasrc, annex_wanted, annex_group, annex_groupwanted, inherit ) if not path: # nothing new was created # TODO is 'notneeded' appropriate in this case? currentds_ap['status'] = 'notneeded' # TODO explain status in 'message' yield currentds_ap yielded.add(currentds_ap['path']) continue remote_repos_to_run_hook_for.append((path, currentds_ap)) # publish web-interface to root dataset on publication server if current_ds.path == ds.path and ui: lgr.info("Uploading web interface to %s" % path) try: CreateSibling.upload_web_interface(path, ssh, shared, ui) except CommandError as e: currentds_ap['status'] = 'error' currentds_ap['message'] = ( "failed to push web interface to the remote datalad repository (%s)", exc_str(e)) yield currentds_ap yielded.add(currentds_ap['path']) continue # in reverse order would be depth first lgr.info("Running post-update hooks in all created siblings") # TODO: add progressbar for path, currentds_ap in remote_repos_to_run_hook_for[::-1]: # Trigger the hook lgr.debug("Running hook for %s (if exists and executable)", path) try: ssh("cd {} " "&& ( [ -x hooks/post-update ] && hooks/post-update || : )" "".format(sh_quote(_path_(path, ".git")))) except CommandError as e: currentds_ap['status'] = 'error' currentds_ap['message'] = ( "failed to run post-update hook under remote path %s (%s)", path, exc_str(e)) yield currentds_ap yielded.add(currentds_ap['path']) continue if not currentds_ap['path'] in yielded: # if we were silent until now everything is just splendid currentds_ap['status'] = 'ok' yield currentds_ap
def __call__( reponame, dataset=None, recursive=False, recursion_limit=None, name='github', existing='error', github_login=None, github_passwd=None, github_organization=None, access_protocol='https', publish_depends=None, dryrun=False): try: # this is an absolute leaf package, import locally to avoid # unecessary dependencies import github as gh except ImportError: raise MissingExternalDependency( 'PyGitHub', msg='GitHub-related functionality is unavailable without this package') # what to operate on ds = require_dataset( dataset, check_installed=True, purpose='create Github sibling') # gather datasets and essential info # dataset instance and mountpoint relative to the top toprocess = [(ds, '')] if recursive: for sub in ds.subdatasets( fulfilled=None, # we want to report on missing dataset in here recursive=recursive, recursion_limit=recursion_limit, result_xfm='datasets'): if not sub.is_installed(): lgr.info('Ignoring unavailable subdataset %s', sub) continue toprocess.append((sub, relpath(sub.path, start=ds.path))) # check for existing remote configuration filtered = [] for d, mp in toprocess: if name in d.repo.get_remotes(): if existing == 'error': msg = '{} already had a configured sibling "{}"'.format( d, name) if dryrun: lgr.error(msg) else: raise ValueError(msg) elif existing == 'skip': continue gh_reponame = '{}{}{}'.format( reponame, '-' if mp else '', template_fx(mp)) filtered.append((d, gh_reponame)) if not filtered: # all skipped return [] # actually make it happen on Github rinfo = _make_github_repos( gh, github_login, github_passwd, github_organization, filtered, existing, access_protocol, dryrun) # lastly configure the local datasets for d, url, existed in rinfo: if not dryrun: # first make sure that annex doesn't touch this one # but respect any existing config ignore_var = 'remote.{}.annex-ignore'.format(name) if not ignore_var in d.config: d.config.add(ignore_var, 'true', where='local') Siblings()( 'configure', dataset=d, name=name, url=url, recursive=False, # TODO fetch=True, maybe only if one existed already publish_depends=publish_depends) # TODO let submodule URLs point to Github (optional) return rinfo
def __call__(sshurl, *, name=None, target_dir=None, target_url=None, target_pushurl=None, dataset=None, recursive=False, recursion_limit=None, existing='error', shared=None, group=None, ui=False, as_common_datasrc=None, publish_by_default=None, publish_depends=None, annex_wanted=None, annex_group=None, annex_groupwanted=None, inherit=False, since=None): if ui: # the webui has been moved to the deprecated extension try: from datalad_deprecated.sibling_webui \ import upload_web_interface except Exception as e: # we could just test for ModuleNotFoundError (which should be # all that would happen with PY3.6+, but be a little more robust # and use the pattern from duecredit if type(e).__name__ not in ('ImportError', 'ModuleNotFoundError'): lgr.error( "Failed to import datalad_deprecated.sibling_webui " "due to %s", str(e)) raise RuntimeError( "The DataLad web UI has been moved to an extension " "package. Please install the Python package " "`datalad_deprecated` to be able to deploy it.") # push uses '^' to annotate the previous pushed committish, and None for default # behavior. '' was/is (to be deprecated) used in `publish` and 'create-sibling'. # Alert user about the mistake if since == '': # deprecation was added prior 0.16.0 import warnings warnings.warn("'since' should point to commitish or use '^'.", DeprecationWarning) since = '^' # # nothing without a base dataset # ds = require_dataset(dataset, check_installed=True, purpose='create sibling(s)') refds_path = ds.path # # all checks that are possible before we start parsing the dataset # if since and not recursive: raise ValueError("The use of 'since' requires 'recursive'") # possibly use sshurl to get the name in case if not specified if not sshurl: if not inherit: raise InsufficientArgumentsError( "needs at least an SSH URL, if no inherit option") if name is None: raise ValueError( "Neither SSH URL, nor the name of sibling to inherit from " "was specified") # It might well be that we already have this remote setup try: sshurl = CreateSibling._get_remote_url(ds, name) except Exception as exc: ce = CapturedException(exc) lgr.debug('%s does not know about url for %s: %s', ds, name, ce) elif inherit: raise ValueError( "For now, for clarity not allowing specifying a custom sshurl " "while inheriting settings") # may be could be safely dropped -- still WiP if not sshurl: # TODO: may be more back up before _prep? super_ds = ds.get_superdataset() if not super_ds: raise ValueError( "Could not determine super dataset for %s to inherit URL" % ds) super_url = CreateSibling._get_remote_url(super_ds, name) # for now assuming hierarchical setup # (TODO: to be able to distinguish between the two, probably # needs storing datalad.*.target_dir to have %RELNAME in there) sshurl = slash_join(super_url, relpath(refds_path, super_ds.path)) # check the login URL sibling_ri = RI(sshurl) ssh_sibling = is_ssh(sibling_ri) if not (ssh_sibling or isinstance(sibling_ri, PathRI)): raise ValueError( "Unsupported SSH URL or path: '{0}', " "use ssh://host/path, host:path or path syntax".format(sshurl)) if not name: name = sibling_ri.hostname if ssh_sibling else "local" lgr.info("No sibling name given. Using %s'%s' as sibling name", "URL hostname " if ssh_sibling else "", name) if since == '^': # consider creating siblings only since the point of # the last update # XXX here we assume one to one mapping of names from local branches # to the remote active_branch = ds.repo.get_active_branch() since = '%s/%s' % (name, active_branch) to_process = [] if recursive: # # parse the base dataset to find all subdatasets that need processing # cand_ds = [ Dataset(r['path']) for r in diff_dataset( ds, fr=since, to='HEAD', # w/o False we might not follow into new subdatasets # which do not have that remote yet setup, # see https://github.com/datalad/datalad/issues/6596 constant_refs=False, # save cycles, we are only looking for datasets annex=None, untracked='no', recursive=True, datasets_only=True, ) # not installed subdatasets would be 'clean' so we would skip them if r.get('type') == 'dataset' and r.get('state', None) != 'clean' ] if not since: # not only subdatasets cand_ds = [ds] + cand_ds else: # only the current ds cand_ds = [ds] # check remotes setup() for d in cand_ds: d_repo = d.repo if d_repo is None: continue checkds_remotes = d.repo.get_remotes() res = dict( action='create_sibling', path=d.path, type='dataset', ) if publish_depends: # make sure dependencies are valid # TODO: inherit -- we might want to automagically create # those dependents as well??? unknown_deps = set( ensure_list(publish_depends)).difference(checkds_remotes) if unknown_deps: yield dict( res, status='error', message=('unknown sibling(s) specified as publication ' 'dependency: %s', unknown_deps), ) continue if name in checkds_remotes and existing in ('error', 'skip'): yield dict( res, sibling_name=name, status='error' if existing == 'error' else 'notneeded', message=( "sibling '%s' already configured (specify alternative " "name, or force reconfiguration via --existing", name), ) continue to_process.append(res) if not to_process: # we ruled out all possibilities # TODO wait for gh-1218 and make better return values lgr.info("No datasets qualify for sibling creation. " "Consider different settings for --existing " "or --since if this is unexpected") return if ssh_sibling: # request ssh connection: lgr.info("Connecting ...") shell = ssh_manager.get_connection(sshurl) else: shell = _RunnerAdapter() sibling_ri.path = str(resolve_path(sibling_ri.path, dataset)) if target_dir: target_dir = opj(sibling_ri.path, target_dir) if target_dir is None: if sibling_ri.path: target_dir = sibling_ri.path else: target_dir = '.' # TODO: centralize and generalize template symbol handling replicate_local_structure = "%RELNAME" not in target_dir if not shell.get_annex_version(): raise MissingExternalDependency( 'git-annex', msg="It's required on the {} machine to create a sibling". format('remote' if ssh_sibling else 'local')) # # all checks done and we have a connection, now do something # # loop over all datasets, ordered from top to bottom to make test # below valid (existing directories would cause the machinery to halt) # But we need to run post-update hook in depth-first fashion, so # would only collect first and then run (see gh #790) yielded = set() remote_repos_to_run_hook_for = [] for currentds_ap in \ sorted(to_process, key=lambda x: x['path'].count('/')): current_ds = Dataset(currentds_ap['path']) path = _create_dataset_sibling( name, current_ds, refds_path, shell, replicate_local_structure, sibling_ri, target_dir, target_url, target_pushurl, existing, shared, group, publish_depends, publish_by_default, ui, as_common_datasrc, annex_wanted, annex_group, annex_groupwanted, inherit) currentds_ap["sibling_name"] = name if not path: # nothing new was created # TODO is 'notneeded' appropriate in this case? currentds_ap['status'] = 'notneeded' # TODO explain status in 'message' yield currentds_ap yielded.add(currentds_ap['path']) continue remote_repos_to_run_hook_for.append((path, currentds_ap)) # publish web-interface to root dataset on publication server if current_ds.path == refds_path and ui: from datalad_deprecated.sibling_webui import upload_web_interface lgr.info("Uploading web interface to %s", path) try: upload_web_interface(path, shell, shared, ui) except CommandError as e: ce = CapturedException(e) currentds_ap['status'] = 'error' currentds_ap['message'] = ( "failed to push web interface to the remote datalad repository (%s)", ce) currentds_ap['exception'] = ce yield currentds_ap yielded.add(currentds_ap['path']) continue # in reverse order would be depth first lgr.info("Running post-update hooks in all created siblings") # TODO: add progressbar for path, currentds_ap in remote_repos_to_run_hook_for[::-1]: # Trigger the hook lgr.debug("Running hook for %s (if exists and executable)", path) try: shell( "cd {} " "&& ( [ -x hooks/post-update ] && hooks/post-update || true )" "".format(sh_quote(_path_(path, ".git")))) except CommandError as e: ce = CapturedException(e) currentds_ap['status'] = 'error' currentds_ap['message'] = ( "failed to run post-update hook under remote path %s (%s)", path, ce) currentds_ap['exception'] = ce yield currentds_ap yielded.add(currentds_ap['path']) continue if not currentds_ap['path'] in yielded: # if we were silent until now everything is just splendid currentds_ap['status'] = 'ok' yield currentds_ap