def truth_load(): truth = dict() # We could use Pandas' Excel functions instead, to save a dependecy, but # this seems perhaps clearer? I may be wrong. book = xlrd.open_workbook(args.truth) for sheet in book.sheets(): if (args.freq == sheet.name): headers = sheet.row_values(0) dates = sheet.col_values(0, start_rowx=1) date_start = to_date(book, dates[0]) date_end = to_date(book, dates[-1]) index = pd.period_range(date_start, periods=len(dates), freq=args.freq) # If this fails, make sure the dates fall on the week start days you # expect. For example, 2010-07-04 is a Sunday. assert (date_start == index[0].to_timestamp(how='start') and date_end == index[-1].to_timestamp(how='start')) for (i, context) in enumerate(headers[1:], start=1): if (len(context) == 0 or context[0] == '_'): # not a real data series, skip continue data = ((j if j != '' else np.nan) for j in sheet.col_values(i, start_rowx=1)) truth[context] = pd.Series(data, index) args.ts_start = index[0].to_timestamp(how='start') args.ts_end = (index[-1].to_timestamp(how='end') + datetime.timedelta(days=1) - datetime.timedelta(microseconds=1)) l.info(' periods: %d' % len(index)) l.info(' start: %s (%s)' % (args.ts_start, args.ts_start.strftime('%A'))) l.info(' end: %s (%s)' % (args.ts_end, args.ts_end.strftime('%A'))) break df = pd.DataFrame(truth, index=index) u.pickle_dump('%s/truth' % args.outdir, df) return df
def truth_load(): truth = dict() # We could use Pandas' Excel functions instead, to save a dependecy, but # this seems perhaps clearer? I may be wrong. book = xlrd.open_workbook(args.truth) for sheet in book.sheets(): if (args.freq == sheet.name): headers = sheet.row_values(0) dates = sheet.col_values(0, start_rowx=1) date_start = to_date(book, dates[0]) date_end = to_date(book, dates[-1]) index = pd.period_range(date_start, periods=len(dates), freq=args.freq) # If this fails, make sure the dates fall on the week start days you # expect. For example, 2010-07-04 is a Sunday. assert ( date_start == index[0].to_timestamp(how='start') and date_end == index[-1].to_timestamp(how='start')) for (i, context) in enumerate(headers[1:], start=1): if (len(context) == 0 or context[0] == '_'): # not a real data series, skip continue data = ((j if j != '' else np.nan) for j in sheet.col_values(i, start_rowx=1)) truth[context] = pd.Series(data, index) args.ts_start = index[0].to_timestamp(how='start') args.ts_end = (index[-1].to_timestamp(how='end') + datetime.timedelta(days=1) - datetime.timedelta(microseconds=1)) l.info(' periods: %d' % len(index)) l.info(' start: %s (%s)' % (args.ts_start, args.ts_start.strftime('%A'))) l.info(' end: %s (%s)' % (args.ts_end, args.ts_end.strftime('%A'))) break df = pd.DataFrame(truth, index=index) u.pickle_dump('%s/truth' % args.outdir, df) return df
def pickle_dump(name, tag, data): if (tag is not None): tag = '.' + tag else: tag = '' u.pickle_dump('%s/%s,%d%s' % (args.out, name, args.distance, tag), data)
def dump(kv): (name, data) = kv u.pickle_dump('%s/%s.%s' % (args_b.value.outdir, name, tag), data)
def dump(self, dir_): u.pickle_dump('%s/model.%d' % (dir_, self.i), self.model) u.pickle_dump('%s/results.%d' % (dir_, self.i), self.results)
def main(self): u.memory_use_log() t_start = time.time() # Replaced with self.cur in __init__ # db = db_glue.DB(self.args.database_file) # assert (db.metadata_get('schema_version') == '5') # normalize start and end times if (self.args.start is None): sql = 'SELECT min(created_at) AS st FROM {0};'.format(self.table) self.cur.execute(sql) self.args.start = self.cur.fetchone()[0] if (self.args.end is None): sql = 'SELECT max(created_at) AS et FROM {0};'.format(self.table) self.cur.execute(sql) # add one second because end time is exclusive self.args.end = self.cur.fetchone()[0] + timedelta(seconds=1) self.args.start = time_.as_utc(self.args.start) self.args.end = time_.as_utc(self.args.end) # print test sequence parameters self.log_parameters() # set up model parameters model_class = u.class_by_name(self.args.model) model_class.parms_init(self.args.model_parms, log_parms=True) # build schedule self.schedule_build(self.args.limit) l.info('scheduled %s tests (%s left over)' % (len(self.schedule), self.args.end - self.schedule[-1].end)) if (not os.path.exists(self.args.output_dir)): os.mkdir(self.args.output_dir) l.info('results in %s' % (self.args.output_dir)) # testing loop for (i, t) in enumerate(self.schedule): if (i+1 < self.args.start_test): l.info('using saved test %d per --start-test' % (i+1)) l.warning('token and tweet counts will be incorrect') # FIXME: hack..... try: t.model = u.Deleted_To_Save_Memory() t.results = u.Deleted_To_Save_Memory() t.i = i t.train_tweet_ct = -1e6 t.train_token_ct = -1e6 t.test_tweet_ct = -1e6 t.unshrink_from_disk(self.args.output_dir, results=True) t.attempted = True except (IOError, x): if (x.errno != 2): raise t.attempted = False else: l.info('starting test %d of %d: %s' % (i+1, len(self.schedule), t)) t.do_test(model_class, self.cur, self.args, i) t.summarize() if (t.attempted): if (self.args.profile_memory): # We dump a memory profile here because it's the high water # mark; we're about to reduce usage significantly. import meliae.scanner as ms filename = 'memory.%d.json' % (i) l.info('dumping memory profile %s' % (filename)) ms.dump_all_objects('%s/%s' % (self.args.output_dir, filename)) t.shrink_to_disk(self.args.output_dir) l.debug('result: %s' % (t.summary)) u.memory_use_log() # done! l.debug('computing summary') self.summarize() l.debug('summary: %s' % (self.summary)) l.debug('saving TSV results') test_indices = u.sl_union_fromtext(len(self.schedule), ':') self.tsv_save_tests('%s/%s' % (self.args.output_dir, 'tests.tsv'), test_indices) l.debug('saving pickled summary') self.memory_use = u.memory_use() self.memory_use_peak = "Not implemented" self.time_use = time.time() - t_start u.pickle_dump('%s/%s' % (self.args.output_dir, 'summary'), self) u.memory_use_log() l.info('done in %s' % (u.fmt_seconds(self.time_use)))
class Test_Sequence(object): def __init__(self, args): self.args = args @property def first_good_test(self): # Any attempted test will give us what we need, but an arbitrary # number of tests might not have been attempted. return next(itertools.ifilter(lambda t: t.attempted, self.schedule)) def main(self): u.memory_use_log() t_start = time.time() db = db_glue.DB(self.args.database_file) l.info('opened database %s' % (self.args.database_file)) assert (db.metadata_get('schema_version') == '5') # normalize start and end times if (self.args.start is None): sql = 'SELECT min(created_at) AS "st [timestamp]" FROM tweet' self.args.start = db.sql(sql)[0]['st'] if (self.args.end is None): sql = 'SELECT max(created_at) AS "et [timestamp]" FROM tweet' # add one second because end time is exclusive self.args.end = db.sql(sql)[0]['et'] + timedelta(seconds=1) self.args.start = time_.as_utc(self.args.start) self.args.end = time_.as_utc(self.args.end) # print test sequence parameters self.log_parameters() # set up model parameters model_class = u.class_by_name(self.args.model) model_class.parms_init(self.args.model_parms, log_parms=True) # build schedule self.schedule_build(self.args.limit) l.info('scheduled %s tests (%s left over)' % (len(self.schedule), self.args.end - self.schedule[-1].end)) if (not os.path.exists(self.args.output_dir)): os.mkdir(self.args.output_dir) l.info('results in %s' % (self.args.output_dir)) # testing loop for (i, t) in enumerate(self.schedule): if (i + 1 < self.args.start_test): l.info('using saved test %d per --start-test' % (i + 1)) l.warning('token and tweet counts will be incorrect') # FIXME: hack..... try: t.model = u.Deleted_To_Save_Memory() t.results = u.Deleted_To_Save_Memory() t.i = i t.train_tweet_ct = -1e6 t.train_token_ct = -1e6 t.test_tweet_ct = -1e6 t.unshrink_from_disk(self.args.output_dir, results=True) t.attempted = True except IOError, x: if (x.errno != 2): raise t.attempted = False else: l.info('starting test %d of %d: %s' % (i + 1, len(self.schedule), t)) t.do_test(model_class, db, self.args, i) t.summarize() if (t.attempted): if (self.args.profile_memory): # We dump a memory profile here because it's the high water # mark; we're about to reduce usage significantly. import meliae.scanner as ms filename = 'memory.%d.json' % (i) l.info('dumping memory profile %s' % (filename)) ms.dump_all_objects('%s/%s' % (self.args.output_dir, filename)) t.shrink_to_disk(self.args.output_dir) l.debug('result: %s' % (t.summary)) u.memory_use_log() # done! l.debug('computing summary') self.summarize() l.debug('summary: %s' % (self.summary)) l.debug('saving TSV results') test_indices = u.sl_union_fromtext(len(self.schedule), ':') self.tsv_save_tests('%s/%s' % (self.args.output_dir, 'tests.tsv'), test_indices) l.debug('saving pickled summary') self.memory_use = u.memory_use() self.memory_use_peak = u.memory_use(True) self.time_use = time.time() - t_start u.pickle_dump('%s/%s' % (self.args.output_dir, 'summary'), self) u.memory_use_log() l.info('done in %s' % (u.fmt_seconds(self.time_use)))