def test_merge(self):
# already have one dataset in self.dataset
dataset = Dataset(path=self.CSV_FILE, connection=self.connection)
result = Dataset.merge([self.dataset, dataset],
connection=self.connection)
self.assertTrue(isinstance(result, Dataset))
self._cleanup(dataset)
self._cleanup(result)
def test_merge(self):
# already have one dataset in self.dataset
dataset = Dataset(path=self.CSV_FILE,
connection=self.connection)
result = Dataset.merge([self.dataset, dataset],
connection=self.connection)
self.assertTrue(isinstance(result, Dataset))
self._cleanup(dataset)
self._cleanup(result)
def test_merge_default_connection(self):
dataset = Dataset(path=self.CSV_FILE,
connection=self.default_connection)
other_dataset = Dataset(path=self.CSV_FILE,
connection=self.default_connection)
result = Dataset.merge([dataset, other_dataset])
self.assertTrue(isinstance(result, Dataset))
self._cleanup(dataset)
self._cleanup(other_dataset)
self._cleanup(result)
def test_merge_default_connection(self):
dataset = Dataset(path=self.CSV_FILE,
connection=self.default_connection)
other_dataset = Dataset(path=self.CSV_FILE,
connection=self.default_connection)
result = Dataset.merge([dataset, other_dataset])
self.assertTrue(isinstance(result, Dataset))
self._cleanup(dataset)
self._cleanup(other_dataset)
self._cleanup(result)
def test_merge_fail(self):
other_dataset = Dataset('12345', connection=self.connection)
result = Dataset.merge([self.dataset, other_dataset],
connection=self.connection)
self.assertFalse(result)
def test_merge_bad_datasets(self):
dataset = {}
other_dataset = []
with self.assertRaises(PyBambooException):
Dataset.merge([dataset, other_dataset], connection=self.connection)
state = dataset.get_info()['state']
print state
bamboo_ids[sector]['originals'][name] = dataset.id
with open(bamboo_id_file, 'wb') as f:
f.write(json.dumps(bamboo_ids))
# merge originals
for sector in bamboo_ids.keys():
if not bamboo_ids[sector]['merged']:
print 'no merged dataset for sector: %s' % sector
datasets = [
Dataset(connection=connection, dataset_id=id)
for name, id in bamboo_ids[sector]['originals'].iteritems()
]
print 'merging datasets: %s' % [dataset.id for dataset in datasets]
merged = Dataset.merge(datasets, connection=connection)
print 'merged: %s' % merged
state = merged.get_info()['state']
while state != 'ready':
time.sleep(1)
state = merged.get_info()['state']
bamboo_ids[sector]['merged'] = merged.id
with open(bamboo_id_file, 'wb') as f:
f.write(json.dumps(bamboo_ids))
# add calculations
print "starting: add calculations"
for sector in bamboo_ids.keys():
print "adding calculation for sector %s" % sector
calculations = []
with open('calculations/%s.txt' % sector) as f:
while state != 'ready':
time.sleep(1)
state = dataset.get_info()['state']
print state
bamboo_ids[sector]['originals'][name] = dataset.id
with open(bamboo_id_file, 'wb') as f:
f.write(json.dumps(bamboo_ids))
# merge originals
for sector in bamboo_ids.keys():
if not bamboo_ids[sector]['merged']:
print 'no merged dataset for sector: %s' % sector
datasets = [Dataset(connection=connection, dataset_id=id) for name, id
in bamboo_ids[sector]['originals'].iteritems()]
print 'merging datasets: %s' % [dataset.id for dataset in datasets]
merged = Dataset.merge(datasets, connection=connection)
print 'merged: %s' % merged
state = merged.get_info()['state']
while state != 'ready':
time.sleep(1)
state = merged.get_info()['state']
bamboo_ids[sector]['merged'] = merged.id
with open(bamboo_id_file, 'wb') as f:
f.write(json.dumps(bamboo_ids))
# add calculations
print "starting: add calculations"
for sector in bamboo_ids.keys():
print "adding calculation for sector %s" % sector
calculations = []
with open('calculations/%s.txt' % sector) as f:
def test_merge_fail(self):
other_dataset = Dataset('12345', connection=self.connection)
result = Dataset.merge([self.dataset, other_dataset],
connection=self.connection)
self.assertFalse(result)
def test_merge_bad_datasets(self):
dataset = {}
other_dataset = []
with self.assertRaises(PyBambooException):
Dataset.merge([dataset, other_dataset],
connection=self.connection)
