def build(data_path: str): data_path = Path(data_path) / 'typos_wiki' fname = data_path / 'misspelings.tsv' if not is_done(data_path): url = 'https://en.wikipedia.org/wiki/Wikipedia:Lists_of_common_misspellings/For_machines' download(fname, url) with fname.open() as f: data = [] for line in f: if line.strip().endswith('<pre>'): break for line in f: if line.strip().startswith('</pre>'): break data.append(line.strip().split('->')) with fname.open('w', newline='') as tsvfile: writer = csv.writer(tsvfile, delimiter='\t') for line in data: writer.writerow(line) mark_done(data_path) print('Built', file=sys.stderr) return fname
def download_resource(url, dest_paths): dest_paths = list(dest_paths) if url.endswith(('.tar.gz', '.gz', '.zip')): download_path = dest_paths[0].parent download_decompress(url, download_path, dest_paths) else: file_name = url.split('/')[-1] dest_files = [dest_path / file_name for dest_path in dest_paths] download(dest_files, url)
def download_resource(url: str, dest_paths: Iterable[Path]) -> None: dest_paths = list(dest_paths) if check_md5(url, dest_paths): log.info(f'Skipped {url} download because of matching hashes') elif url.endswith(('.tar.gz', '.gz', '.zip')): download_path = dest_paths[0].parent download_decompress(url, download_path, dest_paths) else: file_name = url.split('/')[-1] dest_files = [dest_path / file_name for dest_path in dest_paths] download(dest_files, url)
def __init__(self, data_dir: str = '', data_url: str = DB_URL, batch_size: int = None, shuffle: bool = None, seed: int = None, **kwargs): download_dir = expand_path(data_dir) download_path = download_dir.joinpath(data_url.split("/")[-1]) download(download_path, data_url, force_download=False) self.connect = sqlite3.connect(str(download_path), check_same_thread=False) self.db_name = self.get_db_name() self.doc_ids = self.get_doc_ids() self.doc2index = self.map_doc2idx() self.batch_size = batch_size self.shuffle = shuffle self.random = Random(seed)
def build(data_path: str): data_path = Path(data_path) / 'kartaslov' fname = data_path / 'orfo_and_typos.L1_5.csv' if not is_done(data_path): url = 'https://raw.githubusercontent.com/dkulagin/kartaslov/master/dataset/orfo_and_typos/orfo_and_typos.L1_5.csv' download(fname, url) mark_done(data_path) print('Built', file=sys.stderr) return fname
def read(self, data_path, file_name: str='ontonotes_senna.pckl', provide_senna_pos=False, provide_senna_ner=False): path = Path(data_path).resolve() / file_name if not path.exists(): download(str(path), self.URL) with open(path, 'rb') as f: dataset = pickle.load(f) dataset_filtered = {} for key, data in dataset.items(): dataset_filtered[key] = [] for (toks, pos, ner), tags in data: if not provide_senna_pos and not provide_senna_ner: dataset_filtered[key].append((toks, tags)) else: x = [toks] if provide_senna_pos: x.append(pos) if provide_senna_ner: x.append(ner) dataset_filtered[key].append((x, tags)) return dataset_filtered
def build(data_path: str) -> Path: """Download misspellings list from `github <https://raw.githubusercontent.com/dkulagin/kartaslov/master/dataset/orfo_and_typos/orfo_and_typos.L1_5.csv>`_ Args: data_path: target directory to download the data to Returns: path to the resulting csv-file """ data_path = Path(data_path) / 'kartaslov' fname = data_path / 'orfo_and_typos.L1_5.csv' if not is_done(data_path): url = 'https://raw.githubusercontent.com/dkulagin/kartaslov/master/dataset/orfo_and_typos/orfo_and_typos.L1_5.csv' download(fname, url) mark_done(data_path) log.info('Built') return fname
def read(self, data_path, data_types=["train"]): """ Read dataset from data_path directory. Reading files are all data_types + extension (i.e for data_types=["train", "valid"] files "train.csv" and "valid.csv" form data_path will be read) Args: data_path: directory with files data_types: types of considered data (possible: "train", "valid", "test") Returns: dictionary with types from data_types. Each field of dictionary is a list of tuples (x_i, y_i) """ for data_type in data_types: if not Path(data_path).joinpath(data_type + ".csv").exists(): print("Loading {} data from {} to {}".format(data_type, self.url, data_path)) download(source_url=self.url, dest_file_path=Path(data_path).joinpath(data_type + ".csv")) mark_done(data_path) data = {} for data_type in data_types: data[data_type] = pd.read_csv(Path(data_path).joinpath(data_type + ".csv")) new_data = {'train': [], 'valid': [], 'test': []} for field in data_types: for i in range(data[field].shape[0]): new_data[field].append( (data[field].loc[i, 'text'], data[field].loc[i, "intents"].split(","))) return new_data
def download_resources(args): if args.all: urls = ALL_URLS else: urls = REQ_URLS for url in urls: download_path = Path('../download') download_path.mkdir(exist_ok=True) dest_path = download_path embeddings_path = download_path.joinpath('embeddings') if url in EMBEDDING_URLS: embeddings_path.mkdir(exist_ok=True) dest_path = embeddings_path.joinpath(url.split("/")[-1]) download(dest_path, url) elif url in DATA_URLS: dest_path = download_path.joinpath(url.split("/")[-1].split(".")[0]) download_decompress(url, dest_path) else: download_decompress(url, dest_path)
def __init__(self, emb_folder: str, emb_url: str, save_path: str, load_path: str, context_limit: int = 450, question_limit: int = 150, char_limit: int = 16, level: str = 'token', *args, **kwargs): self.emb_folder = expand_path(emb_folder) self.level = level self.emb_url = emb_url self.emb_file_name = Path(emb_url).name self.save_path = expand_path(save_path) self.load_path = expand_path(load_path) self.context_limit = context_limit self.question_limit = question_limit self.char_limit = char_limit self.loaded = False self.NULL = "<NULL>" self.OOV = "<OOV>" self.emb_folder.mkdir(parents=True, exist_ok=True) if not (self.emb_folder / self.emb_file_name).exists(): download(self.emb_folder / self.emb_file_name, self.emb_url) if self.load_path.exists(): self.load()
def read(self, data_path: str, url: str = None, format: str = "csv", class_sep: str = ",", *args, **kwargs) -> dict: """ Read dataset from data_path directory. Reading files are all data_types + extension (i.e for data_types=["train", "valid"] files "train.csv" and "valid.csv" form data_path will be read) Args: data_path: directory with files url: download data files if data_path not exists or empty format: extension of files. Set of Values: ``"csv", "json"`` class_sep: string separator of labels in column with labels sep (str): delimeter for ``"csv"`` files. Default: ``","`` header (int): row number to use as the column names names (array): list of column names to use orient (str): indication of expected JSON string format lines (boolean): read the file as a json object per line. Default: ``False`` Returns: dictionary with types from data_types. Each field of dictionary is a list of tuples (x_i, y_i) """ data_types = ["train", "valid", "test"] train_file = kwargs.get('train', 'train.csv') if not Path(data_path, train_file).exists(): if url is None: raise Exception( "data path {} does not exist or is empty, and download url parameter not specified!" .format(data_path)) log.info("Loading train data from {} to {}".format(url, data_path)) download(source_url=url, dest_file_path=Path(data_path, train_file)) data = {"train": [], "valid": [], "test": []} for data_type in data_types: file_name = kwargs.get(data_type, '{}.{}'.format(data_type, format)) file = Path(data_path).joinpath(file_name) if file.exists(): if format == 'csv': keys = ('sep', 'header', 'names') options = {k: kwargs[k] for k in keys if k in kwargs} df = pd.read_csv(file, **options) elif format == 'json': keys = ('orient', 'lines') options = {k: kwargs[k] for k in keys if k in kwargs} df = pd.read_json(file, **options) else: raise Exception( 'Unsupported file format: {}'.format(format)) x = kwargs.get("x", "text") y = kwargs.get('y', 'labels') if isinstance(x, list): data[data_type] = [([row[x_] for x_ in x], str(row[y]).split(class_sep)) for _, row in df.iterrows()] else: data[data_type] = [(row[x], str(row[y]).split(class_sep)) for _, row in df.iterrows()] else: log.warning("Cannot find {} file".format(file)) return data
def _build_slot_vals(slot_vals_json_path='data/'): url = 'http://files.deeppavlov.ai/datasets/dstc_slot_vals.json' download(slot_vals_json_path, url)
def _download_slot_vals(self): url = 'http://lnsigo.mipt.ru/export/datasets/dstc_slot_vals.json' download(self.save_path, url)
def load(self): url = 'http://lnsigo.mipt.ru/export/datasets/dstc_slot_vals.json' download(self.save_path, url)
def _download_slot_vals(self): url = 'http://files.deeppavlov.ai/datasets/dstc_slot_vals.json' download(self.save_path, url)
def _build_slot_vals(slot_vals_json_path='data/'): url = 'http://files.deeppavlov.ai/datasets/dstc_slot_vals.json' download(slot_vals_json_path, url)
def read(self, data_path: str, url: str = None, format: str = "csv", class_sep: str = ",", *args, **kwargs) -> dict: """ Read dataset from data_path directory. Reading files are all data_types + extension (i.e for data_types=["train", "valid"] files "train.csv" and "valid.csv" form data_path will be read) Args: data_path: directory with files url: download data files if data_path not exists or empty format: extension of files. Set of Values: ``"csv", "json"`` class_sep: string separator of labels in column with labels sep (str): delimeter for ``"csv"`` files. Default: ``","`` header (int): row number to use as the column names names (array): list of column names to use orient (str): indication of expected JSON string format lines (boolean): read the file as a json object per line. Default: ``False`` Returns: dictionary with types from data_types. Each field of dictionary is a list of tuples (x_i, y_i) """ data_types = ["train", "valid", "test"] train_file = kwargs.get('train', 'train.csv') if not Path(data_path, train_file).exists(): if url is None: raise Exception("data path {} does not exist or is empty, and download url parameter not specified!".format(data_path)) log.info("Loading train data from {} to {}".format(url, data_path)) download(source_url=url, dest_file_path=Path(data_path, train_file)) data = {"train": [], "valid": [], "test": []} for data_type in data_types: file_name = kwargs.get(data_type, '{}.{}'.format(data_type, format)) file = Path(data_path).joinpath(file_name) if file.exists(): if format == 'csv': keys = ('sep', 'header', 'names') options = {k: kwargs[k] for k in keys if k in kwargs} df = pd.read_csv(file, **options) elif format == 'json': keys = ('orient', 'lines') options = {k: kwargs[k] for k in keys if k in kwargs} df = pd.read_json(file, **options) else: raise Exception('Unsupported file format: {}'.format(format)) x = kwargs.get("x", "text") y = kwargs.get('y', 'labels') class_sep = kwargs.get('class_sep', ',') if isinstance(x, list): data[data_type] = [([row[x_] for x_ in x], str(row[y]).split(class_sep)) for _, row in df.iterrows()] else: data[data_type] = [(row[x], str(row[y]).split(class_sep)) for _, row in df.iterrows()] else: log.warning("Cannot find {} file".format(file)) return data
def _download_slot_vals(self): url = 'http://files.deeppavlov.ai/datasets/dstc_slot_vals.json' download(self.save_path, url)
def _build_slot_vals(slot_vals_json_path='data/'): url = 'http://lnsigo.mipt.ru/export/datasets/dstc_slot_vals.json' download(slot_vals_json_path, url)
def _build_slot_vals(slot_vals_json_path='data/'): url = 'http://lnsigo.mipt.ru/export/datasets/dstc_slot_vals.json' download(slot_vals_json_path, url)
from deeppavlov.core.data.utils import download import os if not os.path.exists('data/models/glove.txt'): download( 'data/models/glove.txt', source_url='http://files.deeppavlov.ai/embeddings/glove.6B.100d.txt')