def append_data(session, entry: models.Entry, data): entry.append_data(data) nrecords = session.execute( "SELECT count(*) FROM timeseries_1d WHERE entry_id=%d" % entry.id) assert nrecords.scalar() == 450 return True
def import_data(session, entry: models.Entry, data): entry.import_data(data) nrecords = session.execute( 'SELECT count(*) from timeseries_1d where entry_id=%d' % entry.id) assert nrecords.scalar() == 400 return True
def export_to_json_file(entry: Entry, path: str): """ Export to JSON file and verify the file exists """ fpath = os.path.join(path, 'hughes.json') entry.export(path=fpath, fmt='JSON') assert os.path.exists(fpath) return True
def delete_from_internal_table(entry: Entry, datasource: DataSource, delete_source=True, **kwargs): """ Deletes all data from the stored data-table. .. warning:: In case not all data is deleted, you **have to** set :attr:`delete_source` to `False`, otherwise the :class:`DataSource <metacatalog.models.DataSource>` is deleted and the data itself it not reachable anymore. Parameters ---------- entry : Entry The :class:`Entry <metacatalog.models.Entry>`, which requested the deletion. datasource : DataSource The :class:`DataSource <metacatalog.models.DataSource>`, which should be deleted. delete_source : bool If True (default) the :class:`DataSource <metacatalog.models.DataSource>` will be deleted from the database. kwargs : keyword arguments where : str raw SQL filter term to filter the results before deleting """ assert Entry.is_valid(entry) # get the tablename tablename = datasource.path # build the filter filter_term = 'WHERE entry_id=%d' % entry.id if 'where' in kwargs: filter_term = '%s AND %s' % (filter_term, kwargs.get('where')) # build the query sql = 'DELETE FROM %s %s' % (tablename, filter_term) if kwargs.get('verbose', False): print("[SQL]: %s" % sql) # get a session session = object_session(entry) try: session.execute(sql) if delete_source: session.delete(datasource) session.commit() except Exception as e: session.rollback() raise e
def delete_from_local_csv(entry: Entry, datasource: DataSource, delete_source=True, **kwargs): """ Deletes all data from the associated CSV file. .. warning:: In case not all data is deleted, you **have to** set :attr:`delete_source` to `False`, otherwise the :class:`DataSource <metacatalog.models.DataSource>` is deleted and the data itself it not reachable anymore. Parameters ---------- entry : Entry The :class:`Entry <metacatalog.models.Entry>`, which requested the deletion. datasource : DataSource The :class:`DataSource <metacatalog.models.DataSource>`, which should be deleted. delete_source : bool If True (default) the :class:`DataSource <metacatalog.models.DataSource>` will be deleted from the database. kwargs : keyword arguments Will be passed to the CSV read function. """ assert Entry.is_valid(entry) # get the filepath filepath = datasource.path # open the file df = pd.read_csv(filepath, index=None) df.where(df.entry_id != entry.id).dropna(axis='index', inplace=True) # check if other data is left in the dataframe if df.empty: os.remove(filepath) else: df.to_csv(filepath, index=None) # check if the datasource should be deleted if delete_source: try: session = object_session(entry) session.delete(datasource) session.commit() except Exception as e: session.rollback() raise e
def export_to_json(entry: Entry): """ Test the JSON export without data """ json_str = entry.export(path=None, fmt='JSON', no_data=True) assert len(json_str) > 0 # parse back d = json.loads(json_str) # there is the partial entry now, which should be found assert len(d['title']) == 4 return True
def create_datasource(session, entry: models.Entry, data): # create the datasource datasource = entry.create_datasource('timeseries', 'internal', 'timeseries', commit=True) assert datasource is not None # check assert isinstance(datasource, models.DataSource) # build a temporal scale index = data.index datasource.create_scale( resolution=index.inferred_freq, extent=[index[0].isoformat(), index[-1].isoformat()], support=1.0, scale_dimension='temporal') session.add(datasource) session.commit() return True