config = get_config('./db.conf',time_series) outdir = config.outdir psql_connect = '%s dbname=%s' % (config.psql_connect, config.dbname) R = parse_stream(config.resolver) NAMESPACE='http://demi.whoi.edu/mvco/' feed = IfcbFeed(psql_connect) start=strptime('2005-01-01T00:00:00Z',ISO_8601_FORMAT); end=strptime('2014-01-01T00:00:00Z',ISO_8601_FORMAT); with xa(psql_connect) as (c, db): bin_lids = list(feed.between(start,end)) N=8 pids = [] for n in range(N): pid = os.fork() if pid == 0: outfile = os.path.join(outdir,'scores_%d.csv' % n) with open(outfile,'w') as of: for bin_lid in bin_lids[n::N]: bin_pid = NAMESPACE + bin_lid print bin_pid hit = R['class_scores'].resolve(pid=bin_pid) if hit is not None: class_mat = load_class_scores(hit.value)
R = parse_stream(config.resolver) NAMESPACE='http://demi.whoi.edu/mvco/' feed = IfcbFeed(psql_connect) start=strptime('2005-01-01T00:00:00Z',ISO_8601_FORMAT); end=strptime('2014-01-01T00:00:00Z',ISO_8601_FORMAT); s = 'select count(*) from autoclass where bin_lid=%s' q = 'insert into autoclass (bin_lid, class_label, roinums, scores) values (%s, %s, %s, %s)' with xa(psql_connect) as (c, db): n = 0 for bin_lid in feed.between(start,end): bin_pid = NAMESPACE + bin_lid db.execute(s,(bin_lid,)) count = db.fetchone()[0] if count == 0: print 'indexing %s' % bin_pid hit = R['class_scores'].resolve(pid=bin_pid) if hit is not None: class_mat = load_class_scores(hit.value) roinums = {} scores = {} for roinum, label, score in class_scores_mat2class_label_score(class_mat): if not label in roinums: roinums[label] = []