def append_data(session, entry: models.Entry, data):
entry.append_data(data)
nrecords = session.execute(
"SELECT count(*) FROM timeseries_1d WHERE entry_id=%d" % entry.id)
assert nrecords.scalar() == 450
return True
def import_data(session, entry: models.Entry, data):
entry.import_data(data)
nrecords = session.execute(
'SELECT count(*) from timeseries_1d where entry_id=%d' % entry.id)
assert nrecords.scalar() == 400
return True
def export_to_json_file(entry: Entry, path: str):
"""
Export to JSON file and verify the file exists
"""
fpath = os.path.join(path, 'hughes.json')
entry.export(path=fpath, fmt='JSON')
assert os.path.exists(fpath)
return True
def delete_from_internal_table(entry: Entry,
datasource: DataSource,
delete_source=True,
**kwargs):
"""
Deletes all data from the stored data-table.
.. warning::
In case not all data is deleted, you **have to**
set :attr:`delete_source` to `False`, otherwise the
:class:`DataSource <metacatalog.models.DataSource>` is deleted and the data
itself it not reachable anymore.
Parameters
----------
entry : Entry
The :class:`Entry <metacatalog.models.Entry>`, which requested the deletion.
datasource : DataSource
The :class:`DataSource <metacatalog.models.DataSource>`, which should be
deleted.
delete_source : bool
If True (default) the :class:`DataSource <metacatalog.models.DataSource>`
will be deleted from the database.
kwargs : keyword arguments
where : str
raw SQL filter term to filter the results before deleting
"""
assert Entry.is_valid(entry)
# get the tablename
tablename = datasource.path
# build the filter
filter_term = 'WHERE entry_id=%d' % entry.id
if 'where' in kwargs:
filter_term = '%s AND %s' % (filter_term, kwargs.get('where'))
# build the query
sql = 'DELETE FROM %s %s' % (tablename, filter_term)
if kwargs.get('verbose', False):
print("[SQL]: %s" % sql)
# get a session
session = object_session(entry)
try:
session.execute(sql)
if delete_source:
session.delete(datasource)
session.commit()
except Exception as e:
session.rollback()
raise e
def delete_from_local_csv(entry: Entry,
datasource: DataSource,
delete_source=True,
**kwargs):
"""
Deletes all data from the associated CSV file.
.. warning::
In case not all data is deleted, you **have to**
set :attr:`delete_source` to `False`, otherwise the
:class:`DataSource <metacatalog.models.DataSource>` is deleted and the data
itself it not reachable anymore.
Parameters
----------
entry : Entry
The :class:`Entry <metacatalog.models.Entry>`, which requested the deletion.
datasource : DataSource
The :class:`DataSource <metacatalog.models.DataSource>`, which should be
deleted.
delete_source : bool
If True (default) the :class:`DataSource <metacatalog.models.DataSource>`
will be deleted from the database.
kwargs : keyword arguments
Will be passed to the CSV read function.
"""
assert Entry.is_valid(entry)
# get the filepath
filepath = datasource.path
# open the file
df = pd.read_csv(filepath, index=None)
df.where(df.entry_id != entry.id).dropna(axis='index', inplace=True)
# check if other data is left in the dataframe
if df.empty:
os.remove(filepath)
else:
df.to_csv(filepath, index=None)
# check if the datasource should be deleted
if delete_source:
try:
session = object_session(entry)
session.delete(datasource)
session.commit()
except Exception as e:
session.rollback()
raise e
def export_to_json(entry: Entry):
"""
Test the JSON export without data
"""
json_str = entry.export(path=None, fmt='JSON', no_data=True)
assert len(json_str) > 0
# parse back
d = json.loads(json_str)
# there is the partial entry now, which should be found
assert len(d['title']) == 4
return True
def create_datasource(session, entry: models.Entry, data):
# create the datasource
datasource = entry.create_datasource('timeseries',
'internal',
'timeseries',
commit=True)
assert datasource is not None
# check
assert isinstance(datasource, models.DataSource)
# build a temporal scale
index = data.index
datasource.create_scale(
resolution=index.inferred_freq,
extent=[index[0].isoformat(), index[-1].isoformat()],
support=1.0,
scale_dimension='temporal')
session.add(datasource)
session.commit()
return True