def create_test_dataset(project, label_type='class'):
"""Create and yield test labeled dataset, then delete.
Params
------
project : `models.Project` instance
The project under which to create test dataset.
label_type : str
String indicating whether data labels are class names ('class')
for classification, or numerical values for regression (anything other
than 'class'). Defaults to 'class'.
"""
if label_type == 'class':
header = pjoin(os.path.dirname(__file__),
'data', 'asas_training_subset_classes.dat')
elif label_type == 'regr':
header = pjoin(os.path.dirname(__file__),
'data', 'asas_training_subset_targets.dat')
tarball = pjoin(os.path.dirname(__file__),
'data', 'asas_training_subset.tar.gz')
header = shutil.copy2(header, cfg['paths']['upload_folder'])
tarball = shutil.copy2(tarball, cfg['paths']['upload_folder'])
ts_paths = data_management.parse_and_store_ts_data(
tarball, cfg['paths']['ts_data_folder'], header)
d = m.Dataset.add(name='test_ds', project=project, file_uris=ts_paths)
d.save()
try:
yield d
finally:
d.delete_instance()
def test_parsing_and_saving(tmpdir):
data_file_path = pjoin(DATA_PATH, "215153_215176_218272_218934.tar.gz")
header_path = pjoin(DATA_PATH, "215153_215176_218272_218934_metadata.dat")
time_series = data_management.parse_and_store_ts_data(
data_file_path, str(tmpdir), header_path, cleanup_archive=False,
cleanup_header=False)
for ts in time_series:
assert isinstance(ts, str)
assert os.path.exists(ts)
time_series = data_management.parse_and_store_ts_data(
data_file_path, str(tmpdir), None, cleanup_archive=False,
cleanup_header=False)
for ts in time_series:
assert isinstance(ts, str)
assert os.path.exists(ts)
def create_test_dataset(project, label_type='class'):
"""Create and yield test labeled dataset, then delete.
Params
------
project : `models.Project` instance
The project under which to create test dataset.
label_type : str
String indicating whether data labels are class names ('class')
for classification, or numerical values for regression (anything other
than 'class'). Defaults to 'class'.
"""
if label_type == 'class':
header = pjoin(os.path.dirname(__file__),
'data', 'asas_training_subset_classes.dat')
elif label_type == 'regr':
header = pjoin(os.path.dirname(__file__),
'data', 'asas_training_subset_targets.dat')
tarball = pjoin(os.path.dirname(__file__),
'data', 'asas_training_subset.tar.gz')
header = shutil.copy2(header, cfg['paths']['upload_folder'])
tarball = shutil.copy2(tarball, cfg['paths']['upload_folder'])
ts_paths = data_management.parse_and_store_ts_data(
tarball, cfg['paths']['ts_data_folder'], header)
d = m.Dataset.add(name='test_ds', project=project, file_uris=ts_paths)
d.save()
try:
yield d
finally:
d.delete_instance()
def test_parsing_and_saving():
data_file_path = pjoin(DATA_PATH, "215153_215176_218272_218934.tar.gz")
header_path = pjoin(DATA_PATH, "215153_215176_218272_218934_metadata.dat")
time_series = data_management.parse_and_store_ts_data(data_file_path,
TEMP_DIR, header_path, cleanup_archive=False,
cleanup_header=False)
for ts in time_series:
assert all(f in ['meta1', 'meta2', 'meta3']
for f in ts.meta_features.keys())
assert(len(ts.time) == len(ts.measurement)
and len(ts.time) == len(ts.error))
def test_parsing_and_saving():
data_file_path = pjoin(DATA_PATH, "215153_215176_218272_218934.tar.gz")
header_path = pjoin(DATA_PATH, "215153_215176_218272_218934_metadata.dat")
time_series = data_management.parse_and_store_ts_data(
data_file_path,
TEMP_DIR,
header_path,
cleanup_archive=False,
cleanup_header=False)
for ts in time_series:
assert isinstance(ts, str)
assert os.path.exists(ts)
def post(self):
if not 'tarFile' in self.request.files:
return self.error('No tar file uploaded')
zipfile = self.request.files['tarFile'][0]
if zipfile.filename == '':
return self.error('Empty tar file uploaded')
dataset_name = self.get_argument('datasetName')
project_id = self.get_argument('projectID')
zipfile_name = (str(uuid.uuid4()) + "_" +
util.secure_filename(zipfile.filename))
zipfile_path = pjoin(cfg['paths']['upload_folder'], zipfile_name)
with open(zipfile_path, 'wb') as f:
f.write(zipfile['body'])
# Header file is optional for unlabled data w/o metafeatures
if 'headerFile' in self.request.files:
headerfile = self.request.files['headerFile'][0]
headerfile_name = (str(uuid.uuid4()) + "_" +
util.secure_filename(headerfile.filename))
headerfile_path = pjoin(cfg['paths']['upload_folder'], headerfile_name)
with open(headerfile_path, 'wb') as f:
f.write(headerfile['body'])
else:
headerfile_path = None
p = Project.get(Project.id == project_id)
# TODO this should give unique names to the time series files
ts_paths = data_management.parse_and_store_ts_data(
zipfile_path,
cfg['paths']['ts_data_folder'],
headerfile_path)
meta_features = list(time_series.from_netcdf(ts_paths[0])
.meta_features.keys())
unique_ts_paths = [os.path.join(os.path.dirname(ts_path),
str(uuid.uuid4()) + "_" +
util.secure_filename(ts_path))
for ts_path in ts_paths]
for old_path, new_path in zip(ts_paths, unique_ts_paths):
os.rename(old_path, new_path)
file_names = [shorten_fname(ts_path) for ts_path in ts_paths]
d = Dataset.add(name=dataset_name, project=p, file_names=file_names,
file_uris=unique_ts_paths, meta_features=meta_features)
return self.success(d, 'cesium/FETCH_DATASETS')
def test_parsing_and_saving():
data_file_path = pjoin(DATA_PATH, "215153_215176_218272_218934.tar.gz")
header_path = pjoin(DATA_PATH, "215153_215176_218272_218934_metadata.dat")
time_series = data_management.parse_and_store_ts_data(
data_file_path,
TEMP_DIR,
header_path,
cleanup_archive=False,
cleanup_header=False)
for ts in time_series:
assert all(f in ['meta1', 'meta2', 'meta3']
for f in ts.meta_features.keys())
assert (len(ts.time) == len(ts.measurement)
and len(ts.time) == len(ts.error))
def add_files(dataset, create, value, *args, **kwargs):
if not create:
return
if 'class' in dataset.name:
header = pjoin(os.path.dirname(__file__), 'data',
'asas_training_subset_classes.dat')
elif 'regr' in dataset.name:
header = pjoin(os.path.dirname(__file__), 'data',
'asas_training_subset_targets.dat')
else:
header = None
tarball = pjoin(os.path.dirname(__file__), 'data',
'asas_training_subset.tar.gz')
header = shutil.copy2(header, TMP_DIR) if header else None
tarball = shutil.copy2(tarball, TMP_DIR)
ts_paths = data_management.parse_and_store_ts_data(
tarball, TMP_DIR, header)
dataset.files = [DatasetFile(uri=uri) for uri in ts_paths]
DBSession().commit()
def Dataset(dataset_id=None):
"""
"""
if dataset_id:
d = m.Dataset.get(m.Dataset.id == dataset_id)
if not d.project.is_owned_by(get_username()):
raise error('Unauthorized access')
if request.method == 'POST':
form = request.form
if not 'headerFile' in request.files:
return error('No header file uploaded')
if not 'tarFile' in request.files:
return error('No tar file uploaded')
headerfile = request.files['headerFile']
zipfile = request.files['tarFile']
if zipfile.filename == '':
return error('Empty tar file uploaded')
if headerfile.filename == '':
return error('Empty header file uploaded')
dataset_name = form['datasetName']
project_id = form['projectID']
# files have the following attributes:
#
# 'close', 'content_length', 'content_type', 'filename', 'headers',
# 'mimetype', 'mimetype_params', 'name', 'save', 'stream']
# Create unique file names
headerfile_name = (str(uuid.uuid4()) + "_" +
str(secure_filename(headerfile.filename)))
zipfile_name = (str(uuid.uuid4()) + "_" +
str(secure_filename(zipfile.filename)))
headerfile_path = pjoin(cfg['paths']['upload_folder'], headerfile_name)
zipfile_path = pjoin(cfg['paths']['upload_folder'], zipfile_name)
headerfile.save(headerfile_path)
zipfile.save(zipfile_path)
p = m.Project.get(m.Project.id == project_id)
time_series = data_management.parse_and_store_ts_data(
zipfile_path,
cfg['paths']['ts_data_folder'],
headerfile_path)
ts_paths = [ts.path for ts in time_series]
d = m.Dataset.add(name=dataset_name, project=p, file_uris=ts_paths)
return success(d, 'cesium/FETCH_DATASETS')
elif request.method == "GET":
if dataset_id is None:
datasets = [d for p in m.Project.all(get_username())
for d in p.datasets]
else:
datasets = d
return success(datasets)
elif request.method == "DELETE":
if dataset_id is None:
raise error('No dataset specified')
d.delete_instance()
return success(action='cesium/FETCH_DATASETS')
elif request.method == "PUT":
if dataset_id is None:
raise error('No dataset specified')
return error('Dataset updating not yet implemented')
def post(self):
data = self.get_json()
if not 'tarFile' in data:
return self.error('No tar file uploaded')
zipfile = data['tarFile']
tarball_content_types = ('data:application/gzip;base64',
'data:application/x-gzip;base64')
if not zipfile['body'].startswith(tarball_content_types):
return self.error('Invalid tar file - please ensure file is gzip '
'format.')
if zipfile['name'] == '':
return self.error('Empty tar file uploaded')
dataset_name = data['datasetName']
project_id = data['projectID']
zipfile_name = (str(uuid.uuid4()) + "_" +
util.secure_filename(zipfile['name']))
zipfile_path = pjoin(self.cfg['paths:upload_folder'], zipfile_name)
for prefix in tarball_content_types:
zipfile['body'] = zipfile['body'].replace(prefix, '')
with open(zipfile_path, 'wb') as f:
f.write(base64.b64decode(zipfile['body']))
try:
tarfile.open(zipfile_path)
except tarfile.ReadError:
os.remove(zipfile_path)
return self.error('Invalid tar file - please ensure file is gzip '
'format.')
# Header file is optional for unlabled data w/o metafeatures
if 'headerFile' in data:
headerfile = data['headerFile']
headerfile_name = (str(uuid.uuid4()) + "_" +
util.secure_filename(headerfile['name']))
headerfile_path = pjoin(self.cfg['paths:upload_folder'], headerfile_name)
with open(headerfile_path, 'w') as f:
f.write(headerfile['body'])
else:
headerfile_path = None
p = Project.query.filter(Project.id == project_id).one()
ts_paths = data_management.parse_and_store_ts_data(
zipfile_path,
self.cfg['paths:ts_data_folder'],
headerfile_path)
meta_features = list(time_series.load(ts_paths[0]).meta_features.keys())
unique_ts_paths = [os.path.join(os.path.dirname(ts_path),
str(uuid.uuid4()) + "_" +
util.secure_filename(ts_path))
for ts_path in ts_paths]
d = Dataset(name=dataset_name, project=p, meta_features=meta_features)
for old_path, new_path in zip(ts_paths, unique_ts_paths):
os.rename(old_path, new_path)
d.files.append(DatasetFile(name=shorten_fname(old_path),
uri=new_path))
DBSession().add(d)
DBSession().commit()
print(f"\n\n Just added a new dataset ({d}): here's DB contents at this point:",
list(Dataset.query))
return self.success(d.display_info(), 'cesium/FETCH_DATASETS')
def post(self):
data = self.get_json()
if not 'tarFile' in data:
return self.error('No tar file uploaded')
zipfile = data['tarFile']
tarball_content_types = ('data:application/gzip;base64',
'data:application/x-gzip;base64')
if not zipfile['body'].startswith(tarball_content_types):
return self.error('Invalid tar file - please ensure file is gzip '
'format.')
if zipfile['name'] == '':
return self.error('Empty tar file uploaded')
dataset_name = data['datasetName']
project_id = data['projectID']
zipfile_name = (str(uuid.uuid4()) + "_" +
util.secure_filename(zipfile['name']))
zipfile_path = pjoin(self.cfg['paths:upload_folder'], zipfile_name)
for prefix in tarball_content_types:
zipfile['body'] = zipfile['body'].replace(prefix, '')
with open(zipfile_path, 'wb') as f:
f.write(base64.b64decode(zipfile['body']))
try:
tarfile.open(zipfile_path)
except tarfile.ReadError:
os.remove(zipfile_path)
return self.error('Invalid tar file - please ensure file is gzip '
'format.')
# Header file is optional for unlabled data w/o metafeatures
if 'headerFile' in data:
headerfile = data['headerFile']
headerfile_name = (str(uuid.uuid4()) + "_" +
util.secure_filename(headerfile['name']))
headerfile_path = pjoin(self.cfg['paths:upload_folder'],
headerfile_name)
with open(headerfile_path, 'w') as f:
f.write(headerfile['body'])
else:
headerfile_path = None
p = Project.query.filter(Project.id == project_id).one()
ts_paths = data_management.parse_and_store_ts_data(
zipfile_path, self.cfg['paths:ts_data_folder'], headerfile_path)
meta_features = list(
time_series.load(ts_paths[0]).meta_features.keys())
unique_ts_paths = [
os.path.join(
os.path.dirname(ts_path),
str(uuid.uuid4()) + "_" + util.secure_filename(ts_path))
for ts_path in ts_paths
]
d = Dataset(name=dataset_name, project=p, meta_features=meta_features)
for old_path, new_path in zip(ts_paths, unique_ts_paths):
os.rename(old_path, new_path)
d.files.append(
DatasetFile(name=shorten_fname(old_path), uri=new_path))
DBSession().add(d)
DBSession().commit()
return self.success(d.display_info(), 'cesium/FETCH_DATASETS')