def download_data_set(request, data_set_id, what, file_ext): us = get_session(request) data_sets = DataSet.get_common_data_sets() if us is None else us.get_all_allowed_dataset_set() if what == 'reference': data_set_id, file_ext = DataSet.split_seq_ext(data_set_id + file_ext) try: ds = data_sets.get(dirname=data_set_id) except DataSet.DoesNotExist: return HttpResponseNotFound('Data set %s was not found' % data_set_id) if what == 'reference': fpath = os.path.join(ds.get_dirpath(), ds.reference_fname) if DataSet.split_seq_ext(fpath)[1] != file_ext: return HttpResponseNotFound() return __download(fpath, data_set_id + file_ext) else: if what == 'genes': fpath = os.path.join(ds.get_dirpath(), ds.genes_fname) elif what == 'operons': fpath = os.path.join(ds.get_dirpath(), ds.operons_fname) else: return HttpResponseNotFound() if DataSet.split_genes_ext(fpath)[1] != file_ext: return HttpResponseNotFound() return __download(fpath, data_set_id + '_' + what + file_ext)
def __set_data_set_info(qs, response_dict): data_set_title = '' if qs.data_set: if qs.data_set.remember: data_set_title = escape(qs.data_set.name) response_dict['data_set'] = { 'title': data_set_title, 'id': qs.data_set.dirname, 'reference_ext': DataSet.split_seq_ext(qs.data_set.reference_fname)[1] if qs.data_set.reference_fname else None, 'genes_ext': DataSet.split_genes_ext(qs.data_set.genes_fname)[1] if qs.data_set.genes_fname else None, 'operons_ext': DataSet.split_genes_ext(qs.data_set.operons_fname)[1] if qs.data_set.operons_fname else None, }
def as_insertable(self, provider: Provider, dataset: DataSet, data): return { "created_at": datetime.datetime.now(), "provider": provider.serialize(), "dataset": dataset.serialize(), "data": data.serialize() }
def get_data_set(request, data_set_form, user_session, default_name): if data_set_form.cleaned_data['is_created'] is True: name = data_set_form.data['name_created'] def init_folders(data_set): data_set_dirpath = data_set.get_dirpath() #, posted_file.name) if not isdir(data_set_dirpath): os.makedirs(data_set_dirpath) for kind in ['reference', 'genes', 'operons']: posted_file = request.FILES.get(kind) if posted_file: with open(os.path.join(data_set_dirpath, posted_file.name), 'wb+') as f: for chunk in posted_file.chunks(): f.write(chunk) setattr(data_set, kind + '_fname', posted_file.name) if name == '': data_set = DataSet(name=default_name, remember=False, user_session=user_session, user=user_session.user) data_set.save() init_folders(data_set) data_set.save() else: base_name = name i = 1 while DataSet.objects.filter(name=name).exists(): name = base_name + ' (' + str(i) + ')' i += 1 data_set = DataSet(name=name, remember=True, user_session=user_session, user=user_session.user) data_set.save() init_folders(data_set) data_set.save() else: name = data_set_form.data['name_selected'] if name == '' or name == 'no data set': data_set = None else: try: data_set = DataSet.objects.get(name=name) except DataSet.DoesNotExist: logger.error('Data set with name %s does not exits' % name) return HttpResponseBadRequest('Data set does not exist') return data_set
def __init__(self): self.name = "billboard" self.datasets = [ DataSet("hot100", "https://billboard.com/charts/the-billboard-hot-100", datetime.date(1958, 8, 4)) ] self.cookie_header = os.getenv("BILLBOARD_COOKIE_HEADER")
def populate_datasets(app, created_models, verbosity, **kwargs): datasets = DataSet.objects.filter(code='DEMO').all() if len(datasets) == 0: dataset = DataSet() dataset.name = 'DEMO' dataset.code = 'DEMO' dataset.filename = 'inst.json' dataset.size = 440 dataset.save()
def addData(filename): total = 0 f = open('./data/' + filename, "r") if f.mode == 'r': reader = csv.DictReader(f, dialect='excel-tab') for row in reader: revenuePerItem = float(row['Item_price']) * float( row['Item_count']) total = total + revenuePerItem db.session.add( DataInfo(filename, row['Item'], row['Item_description'], row['Item_price'], row['Item_count'], row['Vendor'], row['Vendor_address'])) db.session.add(DataSet(filename, total)) db.session.commit()
def init_dataset(): app.dataset = DataSet.from_csv('datasource.csv')