def download_files(self, folder, local_path, only_use_crux_domains=None): # type: (str, str, bool) -> List[str] """Downloads the resources recursively. Args: folder (str): Crux Dataset Folder from where the file resources should be recursively downloaded. local_path (str): Local OS Path where the file resources should be downloaded. only_use_crux_domains (bool): True if content is required to be downloaded from Crux domains else False. Returns: list (:obj:`str`): List of location of download files. Raises: ValueError: If Folder or local_path is None. OSError: If local_path is an invalid directory location. """ if folder is None: raise ValueError("Folder value shouldn't be empty") if local_path is None: raise ValueError("Local Path value shouldn't be empty") if not os.path.exists(local_path) and not os.path.isdir(local_path): raise OSError("local_path is an invalid directory location") local_file_list = [] # type: List[str] resources = self._list_resources( sort=None, folder=folder, offset=0, limit=None, include_folders=True, model=Resource, ) for resource in resources: resource_path = posixpath.join(folder, resource.name) resource_local_path = os.path.join(local_path, resource.name) if resource.type == "folder": os.mkdir(resource_local_path) log.debug("Created local directory %s", resource_local_path) local_file_list += self.download_files( folder=resource_path, local_path=resource_local_path, only_use_crux_domains=only_use_crux_domains, ) elif resource.type == "file": file_resource = File.from_dict(resource.to_dict(), connection=self.connection) file_resource.download( resource_local_path, only_use_crux_domains=only_use_crux_domains) local_file_list.append(resource_local_path) log.debug("Downloaded file at %s", resource_local_path) return local_file_list
def get_resource_object(resource_type, data, connection=None): # type: (str, Dict[str, Any], CruxClient) -> Union[File, Folder] """Creates resource object based on its type. Args: resource_type (str): Type of resource which needs to be created. data (dict): Dictionary which contains serialized resource data. connection (CruxClient): Connection Object. Defaults to None. Returns: crux.models.Resource: Resource or its Child Object. Raises: TypeError: If it is unable to detect resource type. """ if resource_type == "file": return File.from_dict(data, connection=connection) elif resource_type == "folder": return Folder.from_dict(data, connection=connection) else: raise TypeError("Invalid Resource Type")