def create(cls, s, *sources): composite = add(s, Composite(n_components=0)) for source in sources: add( s, CompositeComponent(input_source=source, output_source=composite)) composite.n_components += 1 return composite
def __topic_journal(self, s, topic): tjournal = s.query(TopicJournal). \ filter(TopicJournal.topic == topic, TopicJournal.date == self._date).one_or_none() if not tjournal: tjournal = add(s, TopicJournal(topic=topic, date=self._date)) return tjournal
def for_modified_files(log, s, paths, callback, owner, force=False): ''' This takes a callback because we need to know whether to mark the file as read or not after processing. The callback should return True on success. The session is used throughout, but not passed to the callback. The callback can contain the same session as internal state or create its own. We avoid open transactions across the callback. ''' for file_path in paths: last_modified = to_time(stat(file_path).st_mtime) hash = md5_hash(file_path) path_scan = s.query(FileScan). \ filter(FileScan.path == file_path, FileScan.owner == owner).one_or_none() # get last scan and make sure it's up-to-date if path_scan: if hash != path_scan.md5_hash: log.warning( 'File at %s appears to have changed since last read on %s') path_scan.md5_hash = hash path_scan.last_scan = 0.0 else: # need to_time here because it's not roundtripped via the database to convert for use below path_scan = add( s, FileScan(path=file_path, owner=owner, md5_hash=hash, last_scan=to_time(0.0))) s.flush() # want this to appear in queries below # only look at hash if we are going to process anyway if force or last_modified > path_scan.last_scan: hash_scan = s.query(FileScan). \ filter(FileScan.md5_hash == hash, FileScan.owner == owner).\ order_by(desc(FileScan.last_scan)).limit(1).one() # must exist as path_scan is a candidate if hash_scan.path != path_scan.path: log.warning('Ignoring duplicate file (details in debug log)') log.debug('%s' % file_path) log.debug('%s' % hash_scan.path) # update the path to avoid triggering in future path_scan.last_scan = hash_scan.last_scan s.commit() if force or last_modified > hash_scan.last_scan: if callback(file_path): log.debug('Marking %s as scanned' % file_path) path_scan.last_scan = last_modified # maybe use now? s.commit() else: log.debug('Not marking %s as scanned' % file_path)
def injury_notes(old_name, new_name): injury_id = next( old.execute('''select id from injury where name like ?''', [old_name]))[0] topic = s.query(Topic).filter(Topic.name == new_name).one() notes = s.query(StatisticName).filter( StatisticName.name == 'Notes', StatisticName.constraint == topic.id).one() for row in old.execute( '''select date, notes from injury_diary where injury_id = ?''', [injury_id]): if row['notes']: # print(row['notes'], len(row['notes'])) tj = add(s, TopicJournal(time=to_time(row['date']), topic=topic)) add( s, StatisticJournalText(statistic_name=notes, source=tj, value=row['notes']))
def test_set(self): with NamedTemporaryFile() as f: args, db = bootstrap_file(f, m(V), '5') bootstrap_file(f, m(V), '5', mm(DEV), configurator=default) with db.session_context() as s: source = add(s, Source()) n = s.query(count(Timestamp.id)).scalar() self.assertEqual(n, 0) Timestamp.set(s, TestTimestamp, source=source) n = s.query(count(Timestamp.id)).scalar() self.assertEqual(n, 1) t = s.query(Timestamp).filter(Timestamp.owner == TestTimestamp).one() self.assertAlmostEqual(t.time.timestamp(), dt.datetime.now().timestamp(), 1)
def add_value(self, s, value, time=None, date=None): from ch2.squeal.utils import add if time and date: raise Exception('Specify one or none of time and date for %s' % self) if not time and not date: time = 0.0 # important this is a float and not an int (or it would be an erroneous date) if date: time = local_date_to_time(date) if time and self.single: raise Exception('%s was given time %s but is not time-variable' % (self, format_time(time))) sjournal = STATISTIC_JOURNAL_CLASSES[self.statistic_name.statistic_journal_type]( statistic_name=self.statistic_name, source=self, value=value, time=time) self.validate(sjournal) return add(s, sjournal)
def filter_modified_files(s, paths, owner, force=False): modified = [] for file_path in paths: last_modified = to_time(stat(file_path).st_mtime) hash = md5_hash(file_path) path_scan = s.query(FileScan). \ filter(FileScan.path == file_path, FileScan.owner == owner).one_or_none() # get last scan and make sure it's up-to-date if path_scan: if hash != path_scan.md5_hash: log.warning( 'File at %s appears to have changed since last read on %s') path_scan.md5_hash = hash path_scan.last_scan = 0.0 else: # need to_time here because it's not roundtripped via the database to convert for use below path_scan = add( s, FileScan(path=file_path, owner=owner, md5_hash=hash, last_scan=to_time(0.0))) s.flush() # want this to appear in queries below # only look at hash if we are going to process anyway if force or last_modified > path_scan.last_scan: hash_scan = s.query(FileScan). \ filter(FileScan.md5_hash == hash, FileScan.owner == owner).\ order_by(desc(FileScan.last_scan)).limit(1).one() # must exist as path_scan is a candidate if hash_scan.path != path_scan.path: log.warning('Ignoring duplicate file (details in debug log)') log.debug('%s' % file_path) log.debug('%s' % hash_scan.path) # update the path to avoid triggering in future path_scan.last_scan = hash_scan.last_scan if force or last_modified > hash_scan.last_scan: modified.append(file_path) s.commit() return modified
def test_sources(self): with NamedTemporaryFile() as f: args, db = bootstrap_file(f, m(V), '5', configurator=acooke) with db.session_context() as s: # add a diary entry diary = s.query(Topic).filter(Topic.name == 'Diary').one() d = add(s, TopicJournal(topic=diary, date='2018-09-29')) d.populate(log, s) self.assertEqual(len(d.topic.fields), 9, list(enumerate(map(str, d.topic.fields)))) self.assertEqual(d.topic.fields[0].statistic_name.name, 'Notes') self.assertEqual(d.topic.fields[1].statistic_name.name, 'Weight', str(d.topic.fields[1])) for field in d.topic.fields: if field in d.statistics: self.assertTrue(d.statistics[field].value is None, field) d.statistics[d.topic.fields[0]].value = 'hello world' d.statistics[d.topic.fields[1]].value = 64.5 with db.session_context() as s: # check the diary entry was persisted diary = s.query(Topic).filter(Topic.name == 'Diary').one() d = s.query(TopicJournal).filter( TopicJournal.topic == diary, TopicJournal.date == '2018-09-29').one() s.flush() d.populate(log, s) self.assertEqual(len(d.topic.fields), 9, list(enumerate(map(str, d.topic.fields)))) self.assertEqual(d.topic.fields[0].statistic_name.name, 'Notes') self.assertEqual(d.statistics[d.topic.fields[0]].value, 'hello world') self.assertEqual(d.topic.fields[1].statistic_name.name, 'Weight') self.assertEqual(d.statistics[d.topic.fields[1]].value, 64.5) self.assertEqual(d.statistics[d.topic.fields[1]].type, StatisticJournalType.FLOAT) # generate summary stats SummaryCalculator(db, schedule='m').run() SummaryCalculator(db, schedule='y').run() with db.session_context() as s: # check the summary stats diary = s.query(Topic).filter(Topic.name == 'Diary').one() weight = s.query(StatisticJournal).join(StatisticName). \ filter(StatisticName.owner == diary, StatisticName.name == 'Weight').one() self.assertEqual(weight.value, 64.5) self.assertEqual(len(weight.measures), 2, weight.measures) self.assertEqual(weight.measures[0].rank, 1) self.assertEqual(weight.measures[0].percentile, 100, weight.measures[0].percentile) n = s.query(count(StatisticJournalFloat.id)).scalar() self.assertEqual(n, 4, n) n = s.query(count(StatisticJournalInteger.id)).scalar() self.assertEqual(n, 11, n) m_avg = s.query(StatisticJournalFloat).join(StatisticName). \ filter(StatisticName.name == 'Avg/Month Weight').one() self.assertEqual(m_avg.value, 64.5) y_avg = s.query(StatisticJournalFloat).join(StatisticName). \ filter(StatisticName.name == 'Avg/Year Weight').one() self.assertEqual(y_avg.value, 64.5) month = s.query(Interval).filter( Interval.schedule == 'm').one() self.assertEqual(month.start, to_date('2018-09-01'), month.start) self.assertEqual(month.finish, to_date('2018-10-01'), month.finish) with db.session_context() as s: # delete the diary entry diary = s.query(Topic).filter(Topic.name == 'Diary').one() d = s.query(TopicJournal).filter( TopicJournal.topic == diary, TopicJournal.date == '2018-09-29').one() s.delete(d) run('sqlite3 %s ".dump"' % f.name, shell=True) with db.session_context() as s: # check the delete cascade self.assertEqual(s.query(count(TopicJournal.id)).scalar(), 0) # this should be zero because the Intervals were automatically deleted for source in s.query(Source).all(): print(source) for journal in s.query(StatisticJournal).all(): print(journal) self.assertEqual( s.query(count(Source.id)).scalar(), 11, list(map(str, s.query(Source).all()))) # constants self.assertEqual( s.query(count(StatisticJournalText.id)).scalar(), 7, s.query(count(StatisticJournalText.id)).scalar()) self.assertEqual( s.query(count(StatisticJournal.id)).scalar(), 7, s.query(count(StatisticJournal.id)).scalar())
fields = dict((field.statistic_name.name, field.statistic_name) for field in diary.fields) notes = fields['Notes'] mood = fields['Mood'] hr = fields['Rest HR'] weight = fields['Weight'] sleep = fields['Sleep'] weather = fields['Weather'] meds = fields['Medication'] for row in old.execute( '''select date, notes, rest_heart_rate, sleep, mood, weather, medication, weight from diary''', []): if row['notes'] or row['mood'] or row['rest_heart_rate'] or row[ 'weight'] or row['sleep'] or row['weather']: tj = add(s, TopicJournal(time=to_time(row['date']), topic=diary)) if row['notes']: add( s, StatisticJournalText(statistic_name=notes, source=tj, value=row['notes'])) if row['mood']: add( s, StatisticJournalInteger(statistic_name=mood, source=tj, value=row['mood'])) if row['rest_heart_rate']: add( s,