def test_merge(self): # already have one dataset in self.dataset dataset = Dataset(path=self.CSV_FILE, connection=self.connection) result = Dataset.merge([self.dataset, dataset], connection=self.connection) self.assertTrue(isinstance(result, Dataset)) self._cleanup(dataset) self._cleanup(result)
def test_merge_default_connection(self): dataset = Dataset(path=self.CSV_FILE, connection=self.default_connection) other_dataset = Dataset(path=self.CSV_FILE, connection=self.default_connection) result = Dataset.merge([dataset, other_dataset]) self.assertTrue(isinstance(result, Dataset)) self._cleanup(dataset) self._cleanup(other_dataset) self._cleanup(result)
def test_merge_fail(self): other_dataset = Dataset('12345', connection=self.connection) result = Dataset.merge([self.dataset, other_dataset], connection=self.connection) self.assertFalse(result)
def test_merge_bad_datasets(self): dataset = {} other_dataset = [] with self.assertRaises(PyBambooException): Dataset.merge([dataset, other_dataset], connection=self.connection)
state = dataset.get_info()['state'] print state bamboo_ids[sector]['originals'][name] = dataset.id with open(bamboo_id_file, 'wb') as f: f.write(json.dumps(bamboo_ids)) # merge originals for sector in bamboo_ids.keys(): if not bamboo_ids[sector]['merged']: print 'no merged dataset for sector: %s' % sector datasets = [ Dataset(connection=connection, dataset_id=id) for name, id in bamboo_ids[sector]['originals'].iteritems() ] print 'merging datasets: %s' % [dataset.id for dataset in datasets] merged = Dataset.merge(datasets, connection=connection) print 'merged: %s' % merged state = merged.get_info()['state'] while state != 'ready': time.sleep(1) state = merged.get_info()['state'] bamboo_ids[sector]['merged'] = merged.id with open(bamboo_id_file, 'wb') as f: f.write(json.dumps(bamboo_ids)) # add calculations print "starting: add calculations" for sector in bamboo_ids.keys(): print "adding calculation for sector %s" % sector calculations = [] with open('calculations/%s.txt' % sector) as f:
while state != 'ready': time.sleep(1) state = dataset.get_info()['state'] print state bamboo_ids[sector]['originals'][name] = dataset.id with open(bamboo_id_file, 'wb') as f: f.write(json.dumps(bamboo_ids)) # merge originals for sector in bamboo_ids.keys(): if not bamboo_ids[sector]['merged']: print 'no merged dataset for sector: %s' % sector datasets = [Dataset(connection=connection, dataset_id=id) for name, id in bamboo_ids[sector]['originals'].iteritems()] print 'merging datasets: %s' % [dataset.id for dataset in datasets] merged = Dataset.merge(datasets, connection=connection) print 'merged: %s' % merged state = merged.get_info()['state'] while state != 'ready': time.sleep(1) state = merged.get_info()['state'] bamboo_ids[sector]['merged'] = merged.id with open(bamboo_id_file, 'wb') as f: f.write(json.dumps(bamboo_ids)) # add calculations print "starting: add calculations" for sector in bamboo_ids.keys(): print "adding calculation for sector %s" % sector calculations = [] with open('calculations/%s.txt' % sector) as f: