def download_data_set(request, data_set_id, what, file_ext):
us = get_session(request)
data_sets = DataSet.get_common_data_sets() if us is None else us.get_all_allowed_dataset_set()
if what == 'reference':
data_set_id, file_ext = DataSet.split_seq_ext(data_set_id + file_ext)
try:
ds = data_sets.get(dirname=data_set_id)
except DataSet.DoesNotExist:
return HttpResponseNotFound('Data set %s was not found' % data_set_id)
if what == 'reference':
fpath = os.path.join(ds.get_dirpath(), ds.reference_fname)
if DataSet.split_seq_ext(fpath)[1] != file_ext:
return HttpResponseNotFound()
return __download(fpath, data_set_id + file_ext)
else:
if what == 'genes':
fpath = os.path.join(ds.get_dirpath(), ds.genes_fname)
elif what == 'operons':
fpath = os.path.join(ds.get_dirpath(), ds.operons_fname)
else:
return HttpResponseNotFound()
if DataSet.split_genes_ext(fpath)[1] != file_ext:
return HttpResponseNotFound()
return __download(fpath, data_set_id + '_' + what + file_ext)
def __set_data_set_info(qs, response_dict):
data_set_title = ''
if qs.data_set:
if qs.data_set.remember:
data_set_title = escape(qs.data_set.name)
response_dict['data_set'] = {
'title': data_set_title,
'id': qs.data_set.dirname,
'reference_ext': DataSet.split_seq_ext(qs.data_set.reference_fname)[1] if qs.data_set.reference_fname else None,
'genes_ext': DataSet.split_genes_ext(qs.data_set.genes_fname)[1] if qs.data_set.genes_fname else None,
'operons_ext': DataSet.split_genes_ext(qs.data_set.operons_fname)[1] if qs.data_set.operons_fname else None,
}
def as_insertable(self, provider: Provider, dataset: DataSet, data):
return {
"created_at": datetime.datetime.now(),
"provider": provider.serialize(),
"dataset": dataset.serialize(),
"data": data.serialize()
}
def get_data_set(request, data_set_form, user_session, default_name):
if data_set_form.cleaned_data['is_created'] is True:
name = data_set_form.data['name_created']
def init_folders(data_set):
data_set_dirpath = data_set.get_dirpath() #, posted_file.name)
if not isdir(data_set_dirpath):
os.makedirs(data_set_dirpath)
for kind in ['reference', 'genes', 'operons']:
posted_file = request.FILES.get(kind)
if posted_file:
with open(os.path.join(data_set_dirpath, posted_file.name), 'wb+') as f:
for chunk in posted_file.chunks():
f.write(chunk)
setattr(data_set, kind + '_fname', posted_file.name)
if name == '':
data_set = DataSet(name=default_name, remember=False, user_session=user_session, user=user_session.user)
data_set.save()
init_folders(data_set)
data_set.save()
else:
base_name = name
i = 1
while DataSet.objects.filter(name=name).exists():
name = base_name + ' (' + str(i) + ')'
i += 1
data_set = DataSet(name=name, remember=True, user_session=user_session, user=user_session.user)
data_set.save()
init_folders(data_set)
data_set.save()
else:
name = data_set_form.data['name_selected']
if name == '' or name == 'no data set':
data_set = None
else:
try:
data_set = DataSet.objects.get(name=name)
except DataSet.DoesNotExist:
logger.error('Data set with name %s does not exits' % name)
return HttpResponseBadRequest('Data set does not exist')
return data_set
def __init__(self):
self.name = "billboard"
self.datasets = [
DataSet("hot100",
"https://billboard.com/charts/the-billboard-hot-100",
datetime.date(1958, 8, 4))
]
self.cookie_header = os.getenv("BILLBOARD_COOKIE_HEADER")
def populate_datasets(app, created_models, verbosity, **kwargs):
datasets = DataSet.objects.filter(code='DEMO').all()
if len(datasets) == 0:
dataset = DataSet()
dataset.name = 'DEMO'
dataset.code = 'DEMO'
dataset.filename = 'inst.json'
dataset.size = 440
dataset.save()
def addData(filename):
total = 0
f = open('./data/' + filename, "r")
if f.mode == 'r':
reader = csv.DictReader(f, dialect='excel-tab')
for row in reader:
revenuePerItem = float(row['Item_price']) * float(
row['Item_count'])
total = total + revenuePerItem
db.session.add(
DataInfo(filename, row['Item'], row['Item_description'],
row['Item_price'], row['Item_count'], row['Vendor'],
row['Vendor_address']))
db.session.add(DataSet(filename, total))
db.session.commit()
def init_dataset():
app.dataset = DataSet.from_csv('datasource.csv')
