def __init__(self, regions, override_d=None): """init object""" GlanceSync.init_logs() self.glancesync = GlanceSync(options_dict=override_d) regions_expanded = list() already_sorted = True for region in regions: if region.endswith(':'): regions_expanded.extend(self.glancesync.get_regions( target=region[:-1])) already_sorted = False else: regions_expanded.append(region) regions = regions_expanded if not regions: regions = self.glancesync.get_regions() already_sorted = False if not already_sorted: regions_unsorted = regions regions = list() for region in self.glancesync.preferable_order: if region in regions_unsorted: regions.append(region) regions_unsorted.remove(region) regions.extend(regions_unsorted) self.regions = regions
def test_delete_master(self): """test delete including 'master:' prefix explicitly""" before = ServersFacade.images['Valladolid'] self.assertIn('001', before.keys()) glancesync = GlanceSync(self.config) self.assertTrue(glancesync.delete_image('master:Valladolid', '001')) self.assertNotIn('001', ServersFacade.images['Valladolid'].keys())
def test_delete_other(self): """test delete with image on other target""" before = ServersFacade.images['other:Madrid'] self.assertIn('201', before.keys()) glancesync = GlanceSync(self.config) self.assertTrue(glancesync.delete_image('other:Madrid', '201')) self.assertNotIn('201', ServersFacade.images['other:Madrid'].keys())
def test_delete(self): """test delete method""" before = ServersFacade.images['Valladolid'] self.assertIn('001', before.keys()) glancesync = GlanceSync(self.config) self.assertTrue(glancesync.delete_image('Valladolid', '001')) self.assertNotIn('001', ServersFacade.images['Valladolid'].keys())
def test_delete_noexists(self): """test delete when the image does not exists""" before = ServersFacade.images['Valladolid'] self.assertNotIn('021', before.keys()) glancesync = GlanceSync(self.config) self.assertFalse(glancesync.delete_image('Valladolid', '021')) self.assertNotIn('021', ServersFacade.images['Valladolid'].keys())
def test_get_regions_include_master(self): """test get_regions with the paremeter to include master""" glancesync = GlanceSync(self.config) result = glancesync.get_regions(omit_master_region=False) result.sort() expected = ['Burgos', 'Valladolid'] expected.sort() self.assertEquals(result, expected)
def test_get_regions(self): """test get_regions method""" glancesync = GlanceSync(self.config) result = glancesync.get_regions() result.sort() expected = ['Burgos'] expected.sort() self.assertEquals(result, expected)
def test_get_images_region_other(self): """test get_images_region with a region on other target""" glancesync = GlanceSync(self.config) result = glancesync.get_images_region('other:Madrid') result.sort(key=lambda image: int(image.id)) expected = self.images_Madrid expected.sort(key=lambda image: int(image.id)) self.assertEquals(len(result), 20) self.assertEquals(result, expected)
def test_get_images_region(self): """test get_images_region with a master region""" glancesync = GlanceSync(self.config) result = glancesync.get_images_region('Valladolid') result.sort(key=lambda image: int(image.id)) expected = self.images_master expected.sort(key=lambda image: int(image.id)) self.assertEquals(len(result), 20) self.assertEquals(result, expected)
def test_get_images_region_master(self): """test get_images_region with a master region, with 'master:' prefix """ glancesync = GlanceSync(self.config) result = glancesync.get_images_region('master:Burgos') result.sort(key=lambda image: int(image.id)) expected = self.images_Burgos expected.sort(key=lambda image: int(image.id)) self.assertEquals(len(result), 20) self.assertEquals(result, expected)
def test_get_regions_other_target(self): """test get_regions with the paremeter target='other'""" glancesync = GlanceSync(self.config) result = glancesync.get_regions(target='other') result.sort() expected = ['other:Madrid', 'other:Region2'] expected.sort() self.assertEquals(result, expected) # omit_master_region does not affect a region different # from master result = glancesync.get_regions(target='other', omit_master_region=False) result.sort() self.assertEquals(result, expected)
def test_get_regions_explicit_master(self): """test get_regions with the paremeter target='master'""" glancesync = GlanceSync(self.config) result = glancesync.get_regions(target='master') result.sort() expected = ['Burgos'] expected.sort() self.assertEquals(result, expected) result = glancesync.get_regions(target='master', omit_master_region=False) result.sort() expected = ['Burgos', 'Valladolid'] expected.sort() self.assertEquals(result, expected)
def test_update_metadata_image_explicit_master(self): """test update master image, with 'master' prefix""" glancesync = GlanceSync(self.config) found = False mock_i = ServersFacade.images for image in self.images_master: if image.id == '001': found = True self.assertIn(image.id, mock_i['Valladolid'].keys()) self.assertTrue(image is not mock_i['Valladolid']['001']) image.user_properties['nid'] = 20 image.user_properties['extra'] = 'new' glancesync.update_metadata_image('master:Valladolid', image) self.assertEquals(image.user_properties, mock_i['Valladolid']['001'].user_properties) self.assertFalse(image.user_properties is mock_i['Valladolid']['001']) self.assertTrue(found)
def test_update_metadata_image_other_target(self): """test update other target image,""" glancesync = GlanceSync(self.config) found = False mock_i = ServersFacade.images for image in self.images_Madrid: if image.id == '201': found = True self.assertIn(image.id, mock_i['other:Madrid'].keys()) self.assertTrue(image is not mock_i['other:Madrid']['201']) image.user_properties['nid'] = 20 image.user_properties['extra'] = 'new' glancesync.update_metadata_image('other:Madrid', image) self.assertEquals(image.user_properties, mock_i['other:Madrid']['201'].user_properties ) self.assertFalse(image.user_properties is mock_i['other:Madrid']['201']) self.assertTrue(found)
def setUp(self): os.environ['GLANCESYNC_USE_MOCK'] = 'True' self.config() self.facade = ServersFacade(dict()) ServersFacade.add_images_from_csv_to_mock(self.path_test) if os.path.exists(self.path_test + '/config'): handler = open(self.path_test + '/config') else: handler = StringIO.StringIO(config1) # self.config = GlanceSyncConfig(stream=handler) self.glancesync = GlanceSync(handler)
def test_backup(self): """test method get_backup""" glancesync = GlanceSync(self.config) self.tmpdir = tempfile.mkdtemp() glancesync.backup_glancemetadata_region('master:Burgos', self.tmpdir) glancesync.backup_glancemetadata_region('Valladolid', self.tmpdir) glancesync.backup_glancemetadata_region('other:Madrid', self.tmpdir) glancesync.backup_glancemetadata_region('other:Region2', self.tmpdir) expected_names = set( ['backup_Valladolid.csv', 'backup_Burgos.csv', 'backup_other:Madrid.csv', 'backup_other:Region2.csv']) found_names = set() for name in glob.glob(self.tmpdir + '/*.csv'): found_names.add(os.path.basename(name)) self.assertItemsEqual(expected_names, found_names) # load csv files to mock and check it is the same old = copy.deepcopy(ServersFacade.images) ServersFacade.clear_mock() ServersFacade.add_images_from_csv_to_mock(self.tmpdir) self.assertEquals(old, ServersFacade.images)
images = list( image for image in imagesregion if image.is_public and ('nid' not in image.user_properties and 'type' in image.user_properties)) images.sort(key=lambda image: image.user_properties['type'] + image.name) _print_images_group(images, properties, comparewith) images = list( image for image in imagesregion if image.is_public and ('nid' in image.user_properties and 'type' not in image.user_properties)) images.sort(key=lambda image: int(image.user_properties['nid'])) _print_images_group(images, properties, comparewith) if __name__ == '__main__': GlanceSync.init_logs() sync_obj = GlanceSync() if len(sys.argv) > 1: regions = sys.argv[1:] else: regions = sync_obj.get_regions() print('======Master (' + sync_obj.master_region + ')') _printimages(sync_obj.master_region_dict.values()) for region in regions: try: print("======" + region) images_region = sync_obj.get_images_region(region) _printimages(images_region, sync_obj.master_region_dict) except Exception: # Don't do anything. Message has been already printed
# don't update if values haven't changed. if (image.user_properties.get('nid', None) == nid and image.user_properties.get('type', None) == typei and image.is_public == public and image.user_properties.get('nid_version', None) == nid_version): continue if nid: image.user_properties['nid'] = nid if typei: image.user_properties['type'] = typei if nid_version: image.user_properties['nid_version'] = nid_version image.is_public = public glancesync.update_metadata_image(region, image) if __name__ == '__main__': glancesync = GlanceSync() for region in glancesync.get_regions(): print 'Updating images on region ' + region try: update_nids(region, glancesync) except Exception: # Do nothing. Error already logged. continue print 'Done'
needs_update = True return needs_update if __name__ == '__main__': if len(sys.argv) < 3: msg = 'Use ' + sys.argv[0] + \ ' <oldname> <newname> [[<region1>] <region2>...]' logging.error(msg) sys.exit(-1) old_name = sys.argv[1] new_name = sys.argv[2] GlanceSync.init_logs() glancesync = GlanceSync() master = glancesync.master_region master_images = glancesync.get_images_region(master) found_with_old_name = None found_with_new_name = None for image in master_images: if image.name == old_name: if found_with_old_name: msg = 'There are more than an image with the old name in the' \ ' master region' logging.error(msg) sys.exit(-1) else:
class TestGlanceSync_Sync(unittest.TestCase): """Basic test: the images are already synchronised. This is also the base class for the following test set. It read a directory with the initial state with .csv files and the same directory with '_result' suffix with the state after the update. Also there are directories .status_pre and .status_post with the results of export_sync_region_status before/after the synchronisation. In self.path_test optionally is also a configuration file with name 'config' """ def config(self): path = os.path.abspath(os.curdir) tmp = get_path(path, RESOURCESPATH) self.path_test = os.path.join(tmp, 'alreadysync') self.regions = ['Valladolid', 'master:Burgos', 'other:Madrid'] def setUp(self): os.environ['GLANCESYNC_USE_MOCK'] = 'True' self.config() self.facade = ServersFacade(dict()) ServersFacade.add_images_from_csv_to_mock(self.path_test) if os.path.exists(self.path_test + '/config'): handler = open(self.path_test + '/config') else: handler = StringIO.StringIO(config1) # self.config = GlanceSyncConfig(stream=handler) self.glancesync = GlanceSync(handler) def tearDown(self): ServersFacade.clear_mock() del os.environ['GLANCESYNC_USE_MOCK'] def test_check_status_pre(self): """test call export_sync_region_status before invoking sync""" path_status = self.path_test + '.status_pre' for region in self.regions: stream = StringIO.StringIO() self.glancesync.\ export_sync_region_status(region, stream) if region.startswith('master:'): region = region[7:] self.assertTrue(os.path.exists(path_status)) f = open(path_status + '/' + region + '.csv', 'rU') expected = f.read().replace('\n', ';') result = stream.getvalue() result = result.replace('\r\n', ';') self.assertEquals(expected, result) def test_sync(self): """test sync_region call and compare the expected results""" for region in self.regions: self.glancesync.sync_region(region) result = copy.deepcopy(ServersFacade.images) ServersFacade.clear_mock() ServersFacade.add_images_from_csv_to_mock(self.path_test + '.result') expected = ServersFacade.images # All the following code is equivalent to: # self.assertEquals(expected[key]), result[key])) # but it is expanded to do debugging of a problem easier. self.assertEquals(len(expected.keys()), len(result.keys())) for key in expected.keys(): self.assertIn(key, result) self.assertEquals(len(expected[key]), len(result[key])) self.assertEquals(set(expected[key].keys()), set(result[key].keys())) for image_key in expected[key].keys(): self.assertEquals(str(expected[key][image_key]), str(result[key][image_key])) def test_check_status_post(self): """run sync_region and then export_sync_region_status. Finally, check these last results""" for region in self.regions: self.glancesync.sync_region(region) path_status = self.path_test + '.status_post' for region in self.regions: stream = StringIO.StringIO() self.glancesync.\ export_sync_region_status(region, stream) if region.startswith('master:'): region = region[7:] self.assertTrue(os.path.exists(path_status)) f = open(path_status + '/' + region + '.csv', 'rU') expected = f.read().rstrip().replace('\n', ';') result = stream.getvalue().rstrip().replace('\r\n', ';') self.assertEquals(expected, result)
class Sync(object): def __init__(self, regions, override_d=None): """init object""" GlanceSync.init_logs() self.glancesync = GlanceSync(options_dict=override_d) regions_expanded = list() already_sorted = True for region in regions: if region.endswith(':'): regions_expanded.extend(self.glancesync.get_regions( target=region[:-1])) already_sorted = False else: regions_expanded.append(region) regions = regions_expanded if not regions: regions = self.glancesync.get_regions() already_sorted = False if not already_sorted: regions_unsorted = regions regions = list() for region in self.glancesync.preferable_order: if region in regions_unsorted: regions.append(region) regions_unsorted.remove(region) regions.extend(regions_unsorted) self.regions = regions def report_status(self): """Report the synchronisation status of the regions""" for region in self.regions: try: stream = StringIO.StringIO() self.glancesync.export_sync_region_status(region, stream) print(stream.getvalue()) except Exception: # Don't do anything. Message has been already printed # try next region continue def parallel_sync(self): """Run the synchronisation in several regions in parallel. The synchronisation inside the region is sequential (i.e. several regions are synchronised simultaneously, but only one image at time is uploaded for each region)""" max_children = self.glancesync.max_children now = datetime.datetime.now() datestr = str(now.year) + str(now.month).zfill(2) + \ str(now.day).zfill(2) + '_' + str(now.hour).zfill(2) +\ str(now.minute).zfill(2) msg = '======Master is ' + self.glancesync.master_region print(msg) sys.stdout.flush() os.mkdir('sync_' + datestr) children = dict() for region in self.regions: try: if len(children) >= max_children: self._wait_child(children) pid = os.fork() if pid > 0: children[pid] = region continue else: path = os.path.join('sync_' + datestr, region + '.txt') handler = logging.FileHandler(path) handler.setFormatter(logging.Formatter('%(message)s')) logger = self.glancesync.log # Remove old handlers for h in logger.handlers: logger.removeHandler(h) logger.addHandler(handler) logger.setLevel(logging.INFO) logger.propagate = 0 self.glancesync.sync_region(region) # After a fork, os_exit() and not sys.exit() must be used. os._exit(0) except Exception: raise sys.stderr.flush() sys.exit(-1) while len(children) > 0: self._wait_child(children) print('All is done.') def sequential_sync(self, dry_run=False): """Run the synchronisation sequentially (that is, do not start the synchronisation to a region before the previous one was completed or failed :param dry_run: if true, do not synchronise images actually """ msg = '======Master is ' + self.glancesync.master_region print(msg) for region in self.regions: try: msg = "======" + region print(msg) sys.stdout.flush() self.glancesync.sync_region(region, dry_run=dry_run) except Exception: # Don't do anything. Message has been already printed # try next region continue def _wait_child(self, children): """ Wait until one of the regions ends its synchronisation and then print the result :param children: :return: a dictionary or regions, indexed by the pid of the process """ finish_direct_child = False while not finish_direct_child: (pid, status) = os.wait() if pid not in children: continue else: finish_direct_child = True if status == 0: msg = 'Region {0} has finished'.format(children[pid]) print(msg) else: msg = 'Region {0} has finished with errors' print(msg.format(children[pid])) del children[pid] sys.stdout.flush() def show_regions(self): """print a full list of the regions available (excluding the master region) in all the targets defined in the configuration file""" regions = self.glancesync.get_regions() for target in self.glancesync.targets.keys(): if target == 'facade' or target == 'master': continue regions.extend(self.glancesync.get_regions(target=target)) print(' '.join(regions)) def make_backup(self): """make a backup of the metadata in the regions specified at the constructor (in addition to the master region). The backup is created in a directory named 'backup_glance_' with the date and time as suffix There is a file for each region (the name is backup_<region>.csv) and inside the file a line for each image. Only the information about public images/ the images owned by the tenant, can be obtained, regardless if the user is an admin. This is a limitation of the glance API""" now = datetime.datetime.now().isoformat() directory = 'backup_glance_' + now os.mkdir(directory) regions = set(self.regions) regions.add(self.glancesync.master_region) for region in regions: try: self.glancesync.backup_glancemetadata_region(region, directory) except Exception: # do nothing. Already logged. continue
# must be deleted needs_update = True return needs_update if __name__ == '__main__': if len(sys.argv) < 3: msg = 'Use ' + sys.argv[0] + \ ' <oldname> <newname> [[<region1>] <region2>...]' logging.error(msg) sys.exit(-1) old_name = sys.argv[1] new_name = sys.argv[2] GlanceSync.init_logs() glancesync = GlanceSync() master = glancesync.master_region master_images = glancesync.get_images_region(master) found_with_old_name = None found_with_new_name = None for image in master_images: if image.name == old_name: if found_with_old_name: msg = 'There are more than an image with the old name in the' \ ' master region' logging.error(msg) sys.exit(-1) else:
import sys import os import logging from glancesync.glancesync import GlanceSync if __name__ == '__main__': confirmation = True if os.environ.get('IKNOWWHATIAMDOING', None) == 'Yes!': confirmation = False if len(sys.argv) != 3 and len(sys.argv) != 2: message = 'Use ' + sys.argv[0] + '<imagename> [<reg1> [<reg2> ...]]' logging.error(message) sys.exit(0) GlanceSync.init_logs() glancesync = GlanceSync() if len(sys.argv) == 3: regions = sys.argv[2:] image_name = sys.argv[1] else: # Obtains the full list or region, including the master region regions = glancesync.get_regions(omit_master_region=False) image_name = sys.argv[1] for region in regions: print("Region: {}".format(region)) try: images = glancesync.get_images_region(region) except Exception: # Don't do anything. Message has been already printed
import os import logging from glancesync.glancesync import GlanceSync if __name__ == '__main__': confirmation = True if os.environ.get('IKNOWWHATIAMDOING', None) == 'Yes!': confirmation = False if len(sys.argv) != 3 and len(sys.argv) != 2: message = 'Use ' + sys.argv[0] + '<imagename> [<reg1> [<reg2> ...]]' logging.error(message) sys.exit(0) GlanceSync.init_logs() glancesync = GlanceSync() if len(sys.argv) == 3: regions = sys.argv[2:] image_name = sys.argv[1] else: # Obtains the full list or region, including the master region regions = glancesync.get_regions(omit_master_region=False) image_name = sys.argv[1] for region in regions: print("Region: {}".format(region)) try: images = glancesync.get_images_region(region) except Exception: # Don't do anything. Message has been already printed
# See the License for the specific language governing permissions and # limitations under the License. # # For those usages not covered by the Apache version 2.0 License please # contact with [email protected] # author = 'jmpr22' import sys import os import datetime from glancesync.glancesync import GlanceSync if __name__ == '__main__': now = datetime.datetime.now().isoformat() glancesync = GlanceSync() glancesync.init_logs() if len(sys.argv) > 1: regions = sys.argv[1:] else: regions = glancesync.get_regions(omit_master_region=False) directory = 'backup_glance_' + now os.mkdir(directory) for region in regions: try: glancesync.backup_glancemetadata_region(region, directory) except Exception: # do nothing. Already logged. continue