def sprint(sprint_ids): ids = sprint_ids.split(',') if request.method == 'GET': sprints = [] snapshots = [] last_snapshots = [] issues = [] committed_issues = [] states = _get_states_dict() for sprint_id in ids: sprint = Sprint.query.filter(Sprint.id == sprint_id).first() sprints.append(sprint) sprint_snapshots = sprint.get_snapshots() snapshots.extend(sprint_snapshots) last_snapshots.append(sprint_snapshots[-1]) # Get all of the issues captured in the latest snapshot issue_snapshots = IssueSnapshot.query.filter(IssueSnapshot.snapshot_id == sprint.last_snapshot.id) issues.extend([(issue.data, issue.state, issue, issue.sprint_count) for issue in issue_snapshots]) committed_issues.extend([commitment.issue_id for commitment in sprint.commitments]) stats_for_all_issues = get_stats_for_snapshots(snapshots) completed_story_points = stats_for_all_issues['completed'][-1] stats = { 'all': stats_for_all_issues, 'committed': get_stats_for_snapshots(snapshots, committed=True), 'label_stats': _get_label_statistics(sprints, completed_story_points), } snapshot_ids = [snapshot.id for snapshot in last_snapshots] issue_state_stats = { 'all': {state: Snapshot.get_points_for_states(snapshot_ids, ids, [state]) for state in states.keys()}, 'committed': {state: Snapshot.get_points_for_states(snapshot_ids, ids, [state], True) for state in states.keys()} } context = { 'sprints': sprints, 'snapshots': snapshots, 'states': states, 'stats': stats, 'issue_state_stats': issue_state_stats, 'issues': sorted(issues, key=lambda x: x[1]), 'committed_issues': committed_issues } return render_template('sprint.html', **context) elif request.method == 'PATCH': if len(ids) > 1: return Response('Can only update one sprint at a time'), httplib.BAD_REQUEST sprint = Sprint.query.filter(Sprint.id == ids[0]).first() for name, value in request.json.items(): if hasattr(sprint, name): setattr(sprint, name, value) db_session.commit() return Response(''), httplib.NO_CONTENT
def __at__(self, index): ''' Get snapshot at specific index ''' key = Snapshot.to_key(index + 1) if key in self.cache: return self.cache.get(key) else: snapshot = Snapshot.by_id(index + 1) if snapshot is not None: self.cache.set(snapshot.key, snapshot.to_dict()) return snapshot.to_dict() return None
def snapshot_issues(repo, milestone): """ Fetches all of the issues for the given sprint and stores them in a database """ sprint = get_or_create_sprint(milestone) print "Processing {} ({})".format(sprint.name, milestone['number']) if sprint.locked is True: print "Skipping '{}', it's locked".format(sprint.name) return snapshot = Snapshot(sprint) db_session.add(snapshot) url = 'https://api.github.com/repos/{}/{}/issues?state=all&milestone={}'.format( settings.ORG, repo, milestone['number']) have_updates = False while True: issues = _auth_get_request(url) for issue in issues.json(): is_updated = save_issue_snapshot(repo, issue, snapshot) have_updates = have_updates or is_updated next_page = _get_next_page(issues) if next_page: url = next_page else: break if have_updates: print "Have updates, committing snapshot" db_session.commit() else: print "No updates, reverting snapshot" db_session.rollback()
def test_submit(self): volley = [("Working - Coding", "Office", { 'happiness': 10, 'stress': 2 }, { 'activity': 'Working', 'activity_sub': "Coding", 'place': "Office" }), ("Working: Meeting", "Office", { 'happiness': 2, 'stress': 4 }, { 'activity': 'Working', 'activity_sub': "Meeting", 'place': "Office" }), ("Running", "Track", { 'happiness': 10, 'stress': 1 }, { 'activity': 'Running', 'activity_sub': None, 'place': "Track" })] for v in volley: activity, place, metrics, expected_vals = v kwargs = {'activity': activity, 'place': place, 'metrics': metrics} sn = Snapshot.Create(self.u, **kwargs) sn.put() for key, val in expected_vals.items(): self.assertEqual(getattr(sn, key), val) for metric, val in metrics.items(): self.assertEqual(sn.get_data_value(metric), val)
def _load(self): ''' Moves snapshots from db into the cache ''' logging.info("Loading game history from database ...") if Snapshot.by_id(1) is None: self.__now__() # Take starting snapshot self.epoch = Snapshot.by_id(1).created try: max_index = len(self) start_index = 1 if len(self) <= 10 else max_index - 9 for index in range(start_index, max_index + 1): snapshot = Snapshot.by_id(index) if not snapshot.key in self.cache: logging.info("Cached snapshot (%d of %d)" % (snapshot.id, max_index)) self.cache.set(snapshot.key, snapshot.to_dict()) logging.info("History load complete.") except KeyboardInterrupt: logging.info("History load stopped by user.")
def import_data(): snapshot = Snapshot() snapshot.snapshot = datetime.now() snapshot.save() print "Importing data from finviz" r = requests.get('http://finviz.com/export.ashx?v=152', cookies={"screenerUrl": "screener.ashx?v=152&f=cap_smallover&ft=4", "customTable": "0,1,2,6,7,10,11,13,14,45,65"}) data = csv_to_dicts(r.text) tickers = [] for row in data: try: stock = Stock() stock.snapshot = snapshot if row["Ticker"]: stock.Ticker = row["Ticker"] print stock.Ticker tickers.append(stock.Ticker) if "Importing " + row["Company"]: stock.Company = row["Company"] if row["Market Cap"]: stock.MarketCap = row["Market Cap"] if row["P/E"]: stock.PE = row["P/E"] if row["P/S"]: stock.PS = row["P/S"] if row["P/B"]: stock.PB = row["P/B"] if row["P/Free Cash Flow"]: stock.PFreeCashFlow = row["P/Free Cash Flow"] if row["Dividend Yield"]: stock.DividendYield = row["Dividend Yield"][:-1] if row["Performance (Half Year)"]: stock.PerformanceHalfYear = row["Performance (Half Year)"][:-1] if row["Price"]: stock.Price = row["Price"] stock.save() except: pdb.set_trace() import_evebitda(snapshot) import_buyback_yield(snapshot)
def _load(self): ''' Moves snapshots from db into the cache ''' logging.info("Loading game history from database ...") if Snapshot.by_id(1) is None: self.__now__() # Take starting snapshot self.epoch = Snapshot.by_id(1).created try: max_index = len(self) start_index = 1 if len(self) <= 10 else max_index - 9 for index in range(start_index, max_index + 1): snapshot = Snapshot.by_id(index) if not snapshot.key in self.cache: logging.info("Cached snapshot (%d of %d)" % ( snapshot.id, max_index )) self.cache.set(snapshot.key, snapshot.to_dict()) logging.info("History load complete.") except KeyboardInterrupt: logging.info("History load stopped by user.")
def get_photos_geo(lat, lon, max_num=1): snapshot_list = [] url = 'https://api.flickr.com/services/rest/?method=flickr.photos.search' querystring = {"api_key": FLICKR_API_KEY, "lat": lat, "lon": lon, "radius": "0.1", "format": "rest"} headers = {'cache-control': "no-cache", } response = requests.request( "GET", url, headers=headers, params=querystring) xml = xmltodict.parse(response.content) # convert xml to json json_string = json.dumps(xml) json_data = json.loads(json_string) counter = 0 try: photos = json_data['rsp']['photos']['photo'] except: return [] for photo in photos: try: id = photo['@id'] farm = photo['@farm'] server = photo['@server'] secret = photo['@secret'] image_uri = construct_uri(id, farm, server, secret) val = get_photo_metadata(id) if val == None: continue tags, date, views = val datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S") s = Snapshot( image_uri=image_uri, views=views, date=datetime_obj, tags=tags) snapshot_list.append(s) except Exception as e: continue counter = counter + 1 if counter == max_num: break return snapshot_list
def snapshot(ctx, board, cycle_time, done): ctx.obj['board_id'] = board ts = TrelloStats(ctx.obj) Snapshot.create_table(fail_silently=True) """ Recording mode - Daily snapshots of a board for ongoing reporting: -> trellis report --board=87hiudhw --cycle-time --spend --revenue --done=Done """ if cycle_time: done_id = ts.get_list_id_from_name(done) cards = ts.get_list_data(done_id) ct = ts.cycle_time(cards) print ct # Create snapshot Snapshot.create(board_id=board, done_id=done_id, cycle_time=ct)
def r_revert(snapshot_id): args_rules = [ Rules.SNAPSHOT_ID.value ] try: ret = dict() ret['state'] = ji.Common.exchange_state(20000) ji.Check.previewing(args_rules, {'snapshot_id': snapshot_id}) snapshot = Snapshot() guest = Guest() snapshot.snapshot_id = snapshot_id snapshot.get_by('snapshot_id') snapshot.progress = 253 snapshot.update() snapshot.get() guest.uuid = snapshot.guest_uuid guest.get_by('uuid') message = { '_object': 'snapshot', 'action': 'revert', 'uuid': guest.uuid, 'snapshot_id': snapshot.snapshot_id, 'node_id': guest.node_id, 'passback_parameters': {'id': snapshot.id} } Utils.emit_instruction(message=json.dumps(message, ensure_ascii=False)) ret['data'] = snapshot.__dict__ return ret except ji.PreviewingError, e: return json.loads(e.message)
def submit(self, d): ''' Submit a snapshot. Assume snapshot is now ''' params = tools.gets(self, strings=['lat', 'lon', 'activity', 'place'], json=['metrics'], lists=['people']) snap = Snapshot.Create(self.user, **params) snap.put() self.success = True self.set_response({'snapshot': snap.json() if snap else None}, message="Snapshot submitted!", debug=True)
def print_stats(sprint_name=None): if sprint_name: sprints = Sprint.query.filter(Sprint.name == sprint_name) else: sprints = Sprint.query.all() for sprint in sprints: print sprint.name snapshot = Snapshot.get_most_recent_for_sprint(sprint) if snapshot is None: print " No stats exist" else: issues = snapshot.issues for state in settings.ISSUE_STATES: print " {}: {}".format(state['label'], _sum_points(issues, state['id']))
def r_update(snapshot_id): snapshot = Snapshot() args_rules = [ Rules.SNAPSHOT_ID.value ] if 'label' in request.json: args_rules.append( Rules.LABEL.value, ) if args_rules.__len__() < 2: ret = dict() ret['state'] = ji.Common.exchange_state(20000) return ret request.json['snapshot_id'] = snapshot_id try: ji.Check.previewing(args_rules, request.json) snapshot.snapshot_id = request.json.get('snapshot_id') snapshot.get_by('snapshot_id') snapshot.label = request.json.get('label', snapshot.label) snapshot.update() snapshot.get() ret = dict() ret['state'] = ji.Common.exchange_state(20000) ret['data'] = snapshot.__dict__ return ret except ji.PreviewingError, e: return json.loads(e.message)
def create_snapshot(self, snapshot_name, before_copy=None): snapshot = Snapshot(snapshot_name=snapshot_name, project_name=self.config['project_name']) self.db.session.add(snapshot) self.db.session.flush() for table_name in self.config['tracked_databases']: if before_copy: before_copy(table_name) table = Table(table_name=table_name, snapshot=snapshot) self.operations.copy_database(table_name, table.get_table_name('master')) self.db.session.add(table) self.db.session.commit() self.start_background_slave_copy(snapshot)
def __now__(self): ''' Returns snapshot object it as a dict ''' snapshot = Snapshot() for team in Team.all(): snapshot_team = SnapshotTeam( team_id=team.id, money=team.money, ) snapshot_team.game_levels = team.game_levels snapshot_team.flags = team.flags dbsession.add(snapshot_team) dbsession.flush() snapshot.teams.append(snapshot_team) dbsession.add(snapshot) dbsession.flush() return snapshot
def __now__(self): ''' Returns snapshot object it as a dict ''' snapshot = Snapshot() bot_manager = BotManager.Instance() for team in Team.all(): snapshot_team = SnapshotTeam(team_id=team.id, money=team.money, bots=bot_manager.count_by_team(team)) snapshot_team.game_levels = team.game_levels snapshot_team.flags = team.flags dbsession.add(snapshot_team) dbsession.flush() snapshot.teams.append(snapshot_team) dbsession.add(snapshot) dbsession.flush() return snapshot
def test_snapshot_calls(self): # Create snap = Snapshot.Create(self.u, activity="Eating", place="Restaurant", people=["Elizabeth"], metrics={'stress': 2}) snap.put() self.assertEqual(snap.get_data_value('stress'), 2) self.assertEqual(snap.activity, "Eating") # List response = self.get_json("/api/snapshot", {}, headers=self.api_headers) snap = response.get('snapshots')[0] print response self.assertEqual(snap.get('activity'), "Eating")
def home(): return render_template("index.html", recent=Snapshot.get_recent())
def __contains__(self, index): return True if Snapshot.by_id(index) is not None else False
def make_instance_snapshot_backup(instance, error, group, provider_class=VolumeProviderBase): LOG.info("Make instance backup for {}".format(instance)) provider = provider_class(instance) infra = instance.databaseinfra database = infra.databases.first() snapshot = Snapshot.create(instance, group, provider.volume) snapshot_final_status = Snapshot.SUCCESS locked = None driver = infra.get_driver() client = None try: client = driver.get_client(instance) locked = lock_instance(driver, instance, client) if not locked: snapshot_final_status = Snapshot.WARNING if 'MySQL' in type(driver).__name__: mysql_binlog_save(client, instance) response = provider.take_snapshot() snapshot.done(response) except Exception as e: errormsg = "Error creating snapshot: {}".format(e) error['errormsg'] = errormsg set_backup_error(infra, snapshot, errormsg) return snapshot finally: if locked: unlock_instance(driver, instance, client) output = {} command = "du -sb /data/.snapshot/%s | awk '{print $1}'" % ( snapshot.snapshot_name ) try: exec_remote_command_host(instance.hostname, command, output) size = int(output['stdout'][0]) snapshot.size = size except Exception as e: snapshot.size = 0 LOG.error("Error exec remote command {}".format(e)) backup_path = database.backup_path if backup_path: now = datetime.now() target_path = "{}/{}/{}/{}/{}".format( backup_path, now.strftime("%Y_%m_%d"), instance.hostname.hostname.split('.')[0], now.strftime("%Y%m%d%H%M%S"), infra.name ) snapshot_path = "/data/.snapshot/{}/data/".format( snapshot.snapshot_name ) output = {} command = """ if [ -d "{backup_path}" ] then rm -rf {backup_path}/20[0-9][0-9]_[0-1][0-9]_[0-3][0-9] & mkdir -p {target_path} cp -r {snapshot_path} {target_path} & fi """.format(backup_path=backup_path, target_path=target_path, snapshot_path=snapshot_path) try: exec_remote_command_host(instance.hostname, command, output) except Exception as e: LOG.error("Error exec remote command {}".format(e)) snapshot.status = snapshot_final_status snapshot.end_at = datetime.now() snapshot.save() register_backup_dbmonitor(infra, snapshot) return snapshot
def resetdb(ctx): Snapshot.drop_table() Snapshot.create_table() click.echo('Snapshots table dropped.')
def make_instance_snapshot_backup(instance, error): LOG.info("Make instance backup for %s" % (instance)) snapshot = Snapshot() snapshot.start_at = datetime.datetime.now() snapshot.type = Snapshot.SNAPSHOPT snapshot.status = Snapshot.RUNNING snapshot.instance = instance snapshot.environment = instance.databaseinfra.environment from dbaas_nfsaas.models import HostAttr as Nfsaas_HostAttr nfsaas_hostattr = Nfsaas_HostAttr.objects.get( host=instance.hostname, is_active=True) snapshot.export_path = nfsaas_hostattr.nfsaas_path databases = Database.objects.filter(databaseinfra=instance.databaseinfra) if databases: snapshot.database_name = databases[0].name snapshot.save() databaseinfra = instance.databaseinfra driver = databaseinfra.get_driver() client = driver.get_client(instance) cloudstack_hostattr = Cloudstack_HostAttr.objects.get( host=instance.hostname) try: LOG.debug('Locking instance %s' % str(instance)) driver.lock_database(client) LOG.debug('Instance %s is locked' % str(instance)) if type(driver).__name__ == 'MySQL': mysql_binlog_save(client, instance, cloudstack_hostattr) nfs_snapshot = NfsaasProvider.create_snapshot(environment=databaseinfra.environment, host=instance.hostname) if 'error' in nfs_snapshot: errormsg = nfs_snapshot['error'] error['errormsg'] = errormsg set_backup_error(databaseinfra, snapshot, errormsg) return False if 'id' in nfs_snapshot and 'snapshot' in nfs_snapshot: snapshot.snapshopt_id = nfs_snapshot['id'] snapshot.snapshot_name = nfs_snapshot['snapshot'] else: errormsg = 'There is no snapshot information' error['errormsg'] = errormsg set_backup_error(databaseinfra, snapshot, errormsg) return False except Exception as e: errormsg = "Error creating snapshot: %s" % (e) error['errormsg'] = errormsg set_backup_error(databaseinfra, snapshot, errormsg) return False finally: LOG.debug('Unlocking instance %s' % str(instance)) driver.unlock_database(client) LOG.debug('Instance %s is unlocked' % str(instance)) output = {} command = "du -sb /data/.snapshot/%s | awk '{print $1}'" % ( snapshot.snapshot_name) try: exec_remote_command(server=instance.hostname.address, username=cloudstack_hostattr.vm_user, password=cloudstack_hostattr.vm_password, command=command, output=output) size = int(output['stdout'][0]) snapshot.size = size except Exception as e: snapshot.size = 0 LOG.error("Error exec remote command %s" % (e)) backup_path = databases[0].backup_path if backup_path: infraname = databaseinfra.name now = datetime.datetime.now() target_path = "{backup_path}/{today_str}/{hostname}/{now_str}/{infraname}".format( backup_path=backup_path, today_str=now.strftime("%Y_%m_%d"), hostname=instance.hostname.hostname.split('.')[0], now_str=now.strftime("%Y%m%d%H%M%S"), infraname=infraname) snapshot_path = "/data/.snapshot/{}/data/".format(snapshot.snapshot_name) output = {} command = """ if [ -d "{backup_path}" ] then rm -rf {backup_path}/20[0-9][0-9]_[0-1][0-12]_[0-3][0-9] mkdir -p {target_path} cp -r {snapshot_path} {target_path} & fi """.format(backup_path=backup_path, target_path=target_path, snapshot_path=snapshot_path) try: exec_remote_command(server=instance.hostname.address, username=cloudstack_hostattr.vm_user, password=cloudstack_hostattr.vm_password, command=command, output=output) except Exception as e: LOG.error("Error exec remote command %s" % (e)) snapshot.status = Snapshot.SUCCESS snapshot.end_at = datetime.datetime.now() snapshot.save() register_backup_dbmonitor(databaseinfra, snapshot) return True
def make_instance_snapshot_backup(instance, error): LOG.info("Make instance backup for %s" % (instance)) snapshot = Snapshot() snapshot.start_at = datetime.datetime.now() snapshot.type=Snapshot.SNAPSHOPT snapshot.status=Snapshot.RUNNING snapshot.instance = instance snapshot.environment = instance.databaseinfra.environment from dbaas_nfsaas.models import HostAttr as Nfsaas_HostAttr nfsaas_hostattr = Nfsaas_HostAttr.objects.get(host=instance.hostname) snapshot.export_path = nfsaas_hostattr.nfsaas_path databases = Database.objects.filter(databaseinfra=instance.databaseinfra) if databases: snapshot.database_name = databases[0].name snapshot.save() databaseinfra = instance.databaseinfra driver = databaseinfra.get_driver() client = driver.get_client(instance) try: driver.lock_database(client) nfs_snapshot = NfsaasProvider.create_snapshot(environment = databaseinfra.environment, plan = databaseinfra.plan, host = instance.hostname) driver.unlock_database(client) if 'error' in nfs_snapshot: errormsg = nfs_snapshot['error'] error['errormsg'] = errormsg set_backup_error(databaseinfra, snapshot, errormsg) return False if 'id' in nfs_snapshot and 'snapshot' in nfs_snapshot: snapshot.snapshopt_id = nfs_snapshot['id'] snapshot.snapshot_name = nfs_snapshot['snapshot'] else: errormsg = 'There is no snapshot information' error['errormsg'] = errormsg set_backup_error(databaseinfra, snapshot, errormsg) return False except Exception, e: errormsg = "Error creating snapshot: %s" % (e) error['errormsg'] = errormsg set_backup_error(databaseinfra, snapshot, errormsg) return False
def list(self, d): limit = self.request.get_range('limit', default=500) snapshots = Snapshot.Recent(self.user, limit=limit) self.set_response({ 'snapshots': [s.json() for s in snapshots if s] }, success=True, debug=True)
def r_create(): args_rules = [ Rules.GUEST_UUID.value ] if 'label' in request.json: args_rules.append( Rules.LABEL.value, ) try: ret = dict() ret['state'] = ji.Common.exchange_state(20000) ji.Check.previewing(args_rules, request.json) snapshot = Snapshot() guest = Guest() guest.uuid = request.json.get('guest_uuid') guest.get_by('uuid') snapshot.label = request.json.get('label', '') snapshot.status = guest.status snapshot.guest_uuid = guest.uuid snapshot.snapshot_id = '_'.join(['tmp', ji.Common.generate_random_code(length=8)]) snapshot.parent_id = '-' snapshot.progress = 0 snapshot.create() snapshot.get_by('snapshot_id') message = { '_object': 'snapshot', 'action': 'create', 'uuid': guest.uuid, 'node_id': guest.node_id, 'passback_parameters': {'id': snapshot.id} } Utils.emit_instruction(message=json.dumps(message, ensure_ascii=False)) ret['data'] = snapshot.__dict__ return ret except ji.PreviewingError, e: return json.loads(e.message)
def r_convert_to_os_template_image(snapshot_id, disk_uuid): args_rules = [ Rules.SNAPSHOT_ID.value, Rules.DISK_UUID.value, Rules.LABEL.value ] try: ret = dict() ret['state'] = ji.Common.exchange_state(20000) ji.Check.previewing(args_rules, {'snapshot_id': snapshot_id, 'disk_uuid': disk_uuid, 'label': request.json.get('label')}) rows, _ = SnapshotDiskMapping.get_by_filter(filter_str=':'.join(['snapshot_id', 'eq', snapshot_id])) disks_uuid = list() for row in rows: disks_uuid.append(row['disk_uuid']) if disk_uuid not in disks_uuid: ret['state'] = ji.Common.exchange_state(40401) ret['state']['sub']['zh-cn'] = ''.join([ret['state']['sub']['zh-cn'], u': 未在快照: ', snapshot_id, u' 中找到磁盘:', disk_uuid]) return ret config = Config() config.id = 1 config.get() snapshot = Snapshot() os_template_image = OSTemplateImage() guest = Guest() disk = Disk() snapshot.snapshot_id = snapshot_id snapshot.get_by('snapshot_id') snapshot.progress = 252 guest.uuid = snapshot.guest_uuid guest.get_by('uuid') disk.uuid = disk_uuid disk.get_by('uuid') os_template_image.id = guest.os_template_image_id os_template_image.get() image_name = '_'.join([snapshot.snapshot_id, disk.uuid]) + '.' + disk.format os_template_image.id = 0 os_template_image.label = request.json.get('label') os_template_image.path = '/'.join([os.path.dirname(os_template_image.path), image_name]) os_template_image.kind = OSTemplateImageKind.custom.value os_template_image.progress = 0 os_template_image.create_time = ji.Common.tus() if os_template_image.exist_by('path'): ret['state'] = ji.Common.exchange_state(40901) ret['state']['sub']['zh-cn'] = ''.join([ret['state']['sub']['zh-cn'], ': ', os_template_image.path]) return ret os_template_image.create() os_template_image.get_by('path') message = { '_object': 'snapshot', 'action': 'convert', 'uuid': disk.guest_uuid, 'snapshot_id': snapshot.snapshot_id, 'storage_mode': config.storage_mode, 'dfs_volume': config.dfs_volume, 'node_id': disk.node_id, 'snapshot_path': disk.path, 'template_path': os_template_image.path, 'os_template_image_id': os_template_image.id, 'passback_parameters': {'id': snapshot.snapshot_id, 'os_template_image_id': os_template_image.id} } Utils.emit_instruction(message=json.dumps(message, ensure_ascii=False)) snapshot.update() return ret except ji.PreviewingError, e: return json.loads(e.message)
def r_content_search(): ret = guest_base.content_search() uuids = list() for guest in ret['data']: uuids.append(guest['uuid']) rows, _ = SSHKeyGuestMapping.get_by_filter(filter_str=':'.join(['guest_uuid', 'in', ','.join(uuids)])) guest_uuid_ssh_key_id_mapping = dict() ssh_keys_id = list() for row in rows: if row['ssh_key_id'] not in ssh_keys_id: ssh_keys_id.append(row['ssh_key_id'].__str__()) if row['guest_uuid'] not in guest_uuid_ssh_key_id_mapping: guest_uuid_ssh_key_id_mapping[row['guest_uuid']] = list() guest_uuid_ssh_key_id_mapping[row['guest_uuid']].append(row['ssh_key_id']) rows, _ = SSHKey.get_by_filter(filter_str=':'.join(['id', 'in', ','.join(ssh_keys_id)])) ssh_key_id_mapping = dict() for row in rows: row['url'] = url_for('v_ssh_keys.show') ssh_key_id_mapping[row['id']] = row rows, _ = Snapshot.get_by_filter(filter_str=':'.join(['guest_uuid', 'in', ','.join(uuids)])) snapshots_guest_uuid_mapping = dict() for row in rows: guest_uuid = row['guest_uuid'] if guest_uuid not in snapshots_guest_uuid_mapping: snapshots_guest_uuid_mapping[guest_uuid] = list() snapshots_guest_uuid_mapping[guest_uuid].append(row) for i, guest in enumerate(ret['data']): guest_uuid = ret['data'][i]['uuid'] if 'ssh_keys' not in ret['data'][i]: ret['data'][i]['ssh_keys'] = list() if guest_uuid in guest_uuid_ssh_key_id_mapping: for ssh_key_id in guest_uuid_ssh_key_id_mapping[guest_uuid]: if ssh_key_id not in ssh_key_id_mapping: continue ret['data'][i]['ssh_keys'].append(ssh_key_id_mapping[ssh_key_id]) if 'snapshot' not in ret['data'][i]: ret['data'][i]['snapshot'] = { 'creatable': True, 'mapping': list() } if guest_uuid in snapshots_guest_uuid_mapping: ret['data'][i]['snapshot']['mapping'] = snapshots_guest_uuid_mapping[guest_uuid] for snapshot in snapshots_guest_uuid_mapping[guest_uuid]: if snapshot['progress'] == 100: continue else: ret['data'][i]['snapshot']['creatable'] = False return ret
def r_delete(snapshots_id): args_rules = [ Rules.SNAPSHOTS_ID.value ] try: ji.Check.previewing(args_rules, {'snapshots_id': snapshots_id}) snapshot = Snapshot() guest = Guest() # 检测所指定的 快照 都存在 for snapshot_id in snapshots_id.split(','): snapshot.snapshot_id = snapshot_id snapshot.get_by('snapshot_id') guest.uuid = snapshot.guest_uuid guest.get_by('uuid') # 执行删除操作 for snapshot_id in snapshots_id.split(','): snapshot.snapshot_id = snapshot_id snapshot.get_by('snapshot_id') guest.uuid = snapshot.guest_uuid guest.get_by('uuid') message = { '_object': 'snapshot', 'action': 'delete', 'uuid': snapshot.guest_uuid, 'snapshot_id': snapshot.snapshot_id, 'node_id': guest.node_id, 'passback_parameters': {'id': snapshot.id} } Utils.emit_instruction(message=json.dumps(message)) # 删除创建失败的 快照 if snapshot.progress == 255: SnapshotDiskMapping.delete_by_filter(filter_str=':'.join(['snapshot_id', 'eq', snapshot.snapshot_id])) snapshot.delete() else: snapshot.progress = 254 snapshot.update() ret = dict() ret['state'] = ji.Common.exchange_state(20000) return ret except ji.PreviewingError, e: return json.loads(e.message)
def make_instance_snapshot_backup(instance, error): LOG.info("Make instance backup for %s" % (instance)) snapshot = Snapshot() snapshot.start_at = datetime.datetime.now() snapshot.type = Snapshot.SNAPSHOPT snapshot.status = Snapshot.RUNNING snapshot.instance = instance snapshot.environment = instance.databaseinfra.environment from dbaas_nfsaas.models import HostAttr as Nfsaas_HostAttr nfsaas_hostattr = Nfsaas_HostAttr.objects.get( host=instance.hostname, is_active=True) snapshot.export_path = nfsaas_hostattr.nfsaas_path databases = Database.objects.filter(databaseinfra=instance.databaseinfra) if databases: snapshot.database_name = databases[0].name snapshot.save() databaseinfra = instance.databaseinfra driver = databaseinfra.get_driver() client = driver.get_client(instance) cloudstack_hostattr = Cloudstack_HostAttr.objects.get( host=instance.hostname) try: LOG.debug('Locking instance %s' % str(instance)) driver.lock_database(client) LOG.debug('Instance %s is locked' % str(instance)) if type(driver).__name__ == 'MySQL': mysql_binlog_save(client, instance, cloudstack_hostattr) nfs_snapshot = NfsaasProvider.create_snapshot(environment=databaseinfra.environment, host=instance.hostname) if 'error' in nfs_snapshot: errormsg = nfs_snapshot['error'] error['errormsg'] = errormsg set_backup_error(databaseinfra, snapshot, errormsg) return False if 'id' in nfs_snapshot and 'snapshot' in nfs_snapshot: snapshot.snapshopt_id = nfs_snapshot['id'] snapshot.snapshot_name = nfs_snapshot['snapshot'] else: errormsg = 'There is no snapshot information' error['errormsg'] = errormsg set_backup_error(databaseinfra, snapshot, errormsg) return False except Exception, e: errormsg = "Error creating snapshot: %s" % (e) error['errormsg'] = errormsg set_backup_error(databaseinfra, snapshot, errormsg) return False
def snapshot_result(guid): snapshot = db.session.query(Snapshot).get(guid) return render_template("snapshot_result.html", snapshot=snapshot, recent=Snapshot.get_recent())
def make_instance_snapshot_backup(instance, error, group, provider_class=VolumeProviderBase): LOG.info("Make instance backup for {}".format(instance)) provider = provider_class(instance) infra = instance.databaseinfra database = infra.databases.first() snapshot = Snapshot.create(instance, group, provider.volume) snapshot_final_status = Snapshot.SUCCESS locked = None driver = infra.get_driver() client = None try: client = driver.get_client(instance) locked = lock_instance(driver, instance, client) if not locked: snapshot_final_status = Snapshot.WARNING if 'MySQL' in type(driver).__name__: mysql_binlog_save(client, instance) response = provider.take_snapshot() snapshot.done(response) except Exception as e: errormsg = "Error creating snapshot: {}".format(e) error['errormsg'] = errormsg set_backup_error(infra, snapshot, errormsg) return snapshot finally: unlock_instance(driver, instance, client) output = {} command = "du -sb /data/.snapshot/%s | awk '{print $1}'" % ( snapshot.snapshot_name) try: exec_remote_command_host(instance.hostname, command, output) size = int(output['stdout'][0]) snapshot.size = size except Exception as e: snapshot.size = 0 LOG.error("Error exec remote command {}".format(e)) backup_path = database.backup_path if backup_path: now = datetime.now() target_path = "{}/{}/{}/{}/{}".format( backup_path, now.strftime("%Y_%m_%d"), instance.hostname.hostname.split('.')[0], now.strftime("%Y%m%d%H%M%S"), infra.name) snapshot_path = "/data/.snapshot/{}/data/".format( snapshot.snapshot_name) output = {} command = """ if [ -d "{backup_path}" ] then rm -rf {backup_path}/20[0-9][0-9]_[0-1][0-9]_[0-3][0-9] & mkdir -p {target_path} cp -r {snapshot_path} {target_path} & fi """.format(backup_path=backup_path, target_path=target_path, snapshot_path=snapshot_path) try: exec_remote_command_host(instance.hostname, command, output) except Exception as e: LOG.error("Error exec remote command {}".format(e)) snapshot.status = snapshot_final_status snapshot.end_at = datetime.now() snapshot.save() register_backup_dbmonitor(infra, snapshot) return snapshot
def make_instance_snapshot_backup(instance, error): LOG.info("Make instance backup for %s" % (instance)) snapshot = Snapshot() snapshot.start_at = datetime.datetime.now() snapshot.type = Snapshot.SNAPSHOPT snapshot.status = Snapshot.RUNNING snapshot.instance = instance snapshot.environment = instance.databaseinfra.environment from dbaas_nfsaas.models import HostAttr as Nfsaas_HostAttr nfsaas_hostattr = Nfsaas_HostAttr.objects.get( host=instance.hostname, is_active=True ) snapshot.export_path = nfsaas_hostattr.nfsaas_path databases = Database.objects.filter(databaseinfra=instance.databaseinfra) if databases: snapshot.database_name = databases[0].name snapshot.save() snapshot_final_status = Snapshot.SUCCESS try: databaseinfra = instance.databaseinfra driver = databaseinfra.get_driver() client = driver.get_client(instance) cloudstack_hostattr = Cloudstack_HostAttr.objects.get( host=instance.hostname ) locked = lock_instance(driver, instance, client) if not locked: snapshot_final_status = Snapshot.WARNING if type(driver).__name__ == 'MySQL': mysql_binlog_save(client, instance, cloudstack_hostattr) nfs_snapshot = create_snapshot( environment=databaseinfra.environment, host=instance.hostname ) if 'id' in nfs_snapshot and 'name' in nfs_snapshot: snapshot.snapshopt_id = nfs_snapshot['id'] snapshot.snapshot_name = nfs_snapshot['name'] else: errormsg = 'There is no snapshot information' error['errormsg'] = errormsg set_backup_error(databaseinfra, snapshot, errormsg) return snapshot except Exception as e: errormsg = "Error creating snapshot: %s" % (e) error['errormsg'] = errormsg set_backup_error(databaseinfra, snapshot, errormsg) return snapshot finally: if locked: unlock_instance(driver, instance, client) output = {} command = "du -sb /data/.snapshot/%s | awk '{print $1}'" % ( snapshot.snapshot_name) try: exec_remote_command(server=instance.hostname.address, username=cloudstack_hostattr.vm_user, password=cloudstack_hostattr.vm_password, command=command, output=output) size = int(output['stdout'][0]) snapshot.size = size except Exception as e: snapshot.size = 0 LOG.error("Error exec remote command %s" % (e)) backup_path = databases[0].backup_path if backup_path: infraname = databaseinfra.name now = datetime.datetime.now() target_path = "{backup_path}/{today_str}/{hostname}/{now_str}/{infraname}".format( backup_path=backup_path, today_str=now.strftime("%Y_%m_%d"), hostname=instance.hostname.hostname.split('.')[0], now_str=now.strftime("%Y%m%d%H%M%S"), infraname=infraname) snapshot_path = "/data/.snapshot/{}/data/".format(snapshot.snapshot_name) output = {} command = """ if [ -d "{backup_path}" ] then rm -rf {backup_path}/20[0-9][0-9]_[0-1][0-9]_[0-3][0-9] & mkdir -p {target_path} cp -r {snapshot_path} {target_path} & fi """.format(backup_path=backup_path, target_path=target_path, snapshot_path=snapshot_path) try: exec_remote_command(server=instance.hostname.address, username=cloudstack_hostattr.vm_user, password=cloudstack_hostattr.vm_password, command=command, output=output) except Exception as e: LOG.error("Error exec remote command %s" % (e)) snapshot.status = snapshot_final_status snapshot.end_at = datetime.datetime.now() snapshot.save() register_backup_dbmonitor(databaseinfra, snapshot) return snapshot
def make_instance_snapshot_backup(instance, error, group, provider_class=VolumeProviderSnapshot, target_volume=None, current_hour=None): LOG.info("Make instance backup for {}".format(instance)) provider = provider_class(instance) infra = instance.databaseinfra database = infra.databases.first() snapshot = Snapshot.create( instance, group, target_volume or provider.volume, environment=provider.environment ) snapshot_final_status = Snapshot.SUCCESS locked = None driver = infra.get_driver() client = None try: client = driver.get_client(instance) locked = lock_instance(driver, instance, client) if not locked: snapshot_final_status = Snapshot.WARNING if 'MySQL' in type(driver).__name__: mysql_binlog_save(client, instance) current_time = datetime.now() has_snapshot = Snapshot.objects.filter( status=Snapshot.WARNING, instance=instance, end_at__year=current_time.year, end_at__month=current_time.month, end_at__day=current_time.day ) backup_hour_list = Configuration.get_by_name_as_list( 'make_database_backup_hour' ) if (snapshot_final_status == Snapshot.WARNING and has_snapshot): if str(current_hour) in backup_hour_list: raise Exception( "Backup with WARNING already created today." ) else: response = provider.take_snapshot() snapshot.done(response) snapshot.save() except Exception as e: errormsg = "Error creating snapshot: {}".format(e) error['errormsg'] = errormsg set_backup_error(infra, snapshot, errormsg) return snapshot finally: unlock_instance(driver, instance, client) if not snapshot.size: command = "du -sb /data/.snapshot/%s | awk '{print $1}'" % ( snapshot.snapshot_name ) try: output = instance.hostname.ssh.run_script(command) size = int(output['stdout'][0]) snapshot.size = size except Exception as e: snapshot.size = 0 LOG.error("Error exec remote command {}".format(e)) backup_path = database.backup_path if backup_path: now = datetime.now() target_path = "{}/{}/{}/{}/{}".format( backup_path, now.strftime("%Y_%m_%d"), instance.hostname.hostname.split('.')[0], now.strftime("%Y%m%d%H%M%S"), infra.name ) snapshot_path = "/data/.snapshot/{}/data/".format( snapshot.snapshot_name ) command = """ if [ -d "{backup_path}" ] then rm -rf {backup_path}/20[0-9][0-9]_[0-1][0-9]_[0-3][0-9] & mkdir -p {target_path} cp -r {snapshot_path} {target_path} & fi """.format(backup_path=backup_path, target_path=target_path, snapshot_path=snapshot_path) try: instance.hostname.ssh.run_script(command) except Exception as e: LOG.error("Error exec remote command {}".format(e)) snapshot.status = snapshot_final_status snapshot.end_at = datetime.now() snapshot.save() register_backup_dbmonitor(infra, snapshot) return snapshot
def response_processor(cls): _object = cls.message['message']['_object'] action = cls.message['message']['action'] uuid = cls.message['message']['uuid'] state = cls.message['type'] data = cls.message['message']['data'] node_id = cls.message['node_id'] if _object == 'guest': if action == 'create': if state == ResponseState.success.value: # 系统盘的 UUID 与其 Guest 的 UUID 相同 cls.disk.uuid = uuid cls.disk.get_by('uuid') cls.disk.guest_uuid = uuid cls.disk.state = DiskState.mounted.value # disk_info['virtual-size'] 的单位为Byte,需要除以 1024 的 3 次方,换算成单位为 GB 的值 cls.disk.size = data['disk_info']['virtual-size'] / (1024** 3) cls.disk.update() else: cls.guest.uuid = uuid cls.guest.get_by('uuid') cls.guest.status = GuestState.dirty.value cls.guest.update() elif action == 'migrate': pass elif action == 'delete': if state == ResponseState.success.value: cls.config.get() cls.guest.uuid = uuid cls.guest.get_by('uuid') if IP(cls.config.start_ip).int() <= IP( cls.guest.ip).int() <= IP(cls.config.end_ip).int(): if db.r.srem(app.config['ip_used_set'], cls.guest.ip): db.r.sadd(app.config['ip_available_set'], cls.guest.ip) if (cls.guest.vnc_port - cls.config.start_vnc_port) <= \ (IP(cls.config.end_ip).int() - IP(cls.config.start_ip).int()): if db.r.srem(app.config['vnc_port_used_set'], cls.guest.vnc_port): db.r.sadd(app.config['vnc_port_available_set'], cls.guest.vnc_port) cls.guest.delete() # TODO: 加入是否删除使用的数据磁盘开关,如果为True,则顺便删除使用的磁盘。否则解除该磁盘被使用的状态。 cls.disk.uuid = uuid cls.disk.get_by('uuid') cls.disk.delete() cls.disk.update_by_filter( { 'guest_uuid': '', 'sequence': -1, 'state': DiskState.idle.value }, filter_str='guest_uuid:eq:' + cls.guest.uuid) SSHKeyGuestMapping.delete_by_filter(filter_str=':'.join( ['guest_uuid', 'eq', cls.guest.uuid])) elif action == 'reset_password': if state == ResponseState.success.value: cls.guest.uuid = uuid cls.guest.get_by('uuid') cls.guest.password = cls.message['message'][ 'passback_parameters']['password'] cls.guest.update() elif action == 'attach_disk': cls.disk.uuid = cls.message['message']['passback_parameters'][ 'disk_uuid'] cls.disk.get_by('uuid') if state == ResponseState.success.value: cls.disk.guest_uuid = uuid cls.disk.sequence = cls.message['message'][ 'passback_parameters']['sequence'] cls.disk.state = DiskState.mounted.value cls.disk.update() elif action == 'detach_disk': cls.disk.uuid = cls.message['message']['passback_parameters'][ 'disk_uuid'] cls.disk.get_by('uuid') if state == ResponseState.success.value: cls.disk.guest_uuid = '' cls.disk.sequence = -1 cls.disk.state = DiskState.idle.value cls.disk.update() elif action == 'boot': if state == ResponseState.success.value: pass elif _object == 'disk': if action == 'create': cls.disk.uuid = uuid cls.disk.get_by('uuid') cls.disk.node_id = node_id if state == ResponseState.success.value: cls.disk.state = DiskState.idle.value else: cls.disk.state = DiskState.dirty.value cls.disk.update() elif action == 'resize': if state == ResponseState.success.value: cls.config.get() cls.disk.uuid = uuid cls.disk.get_by('uuid') cls.disk.size = cls.message['message'][ 'passback_parameters']['size'] cls.disk.quota(config=cls.config) cls.disk.update() elif action == 'delete': cls.disk.uuid = uuid cls.disk.get_by('uuid') cls.disk.delete() elif _object == 'snapshot': if action == 'create': cls.snapshot.id = cls.message['message'][ 'passback_parameters']['id'] cls.snapshot.get() if state == ResponseState.success.value: cls.snapshot.snapshot_id = data['snapshot_id'] cls.snapshot.parent_id = data['parent_id'] cls.snapshot.xml = data['xml'] cls.snapshot.progress = 100 cls.snapshot.update() disks, _ = Disk.get_by_filter(filter_str='guest_uuid:eq:' + cls.snapshot.guest_uuid) for disk in disks: cls.snapshot_disk_mapping.snapshot_id = cls.snapshot.snapshot_id cls.snapshot_disk_mapping.disk_uuid = disk['uuid'] cls.snapshot_disk_mapping.create() else: cls.snapshot.progress = 255 cls.snapshot.update() if action == 'delete': if state == ResponseState.success.value: cls.snapshot.id = cls.message['message'][ 'passback_parameters']['id'] cls.snapshot.get() # 更新子快照的 parent_id 为,当前快照的 parent_id。因为当前快照已被删除。 Snapshot.update_by_filter( {'parent_id': cls.snapshot.parent_id}, filter_str='parent_id:eq:' + cls.snapshot.snapshot_id) SnapshotDiskMapping.delete_by_filter(filter_str=':'.join( ['snapshot_id', 'eq', cls.snapshot.snapshot_id])) cls.snapshot.delete() else: pass if action == 'revert': # 不论恢复成功与否,都使快照恢复至正常状态。 cls.snapshot.id = cls.message['message'][ 'passback_parameters']['id'] cls.snapshot.get() cls.snapshot.progress = 100 cls.snapshot.update() if action == 'convert': cls.snapshot.snapshot_id = cls.message['message'][ 'passback_parameters']['id'] cls.snapshot.get_by('snapshot_id') cls.snapshot.progress = 100 cls.snapshot.update() cls.os_template_image.id = cls.message['message'][ 'passback_parameters']['os_template_image_id'] cls.os_template_image.get() if state == ResponseState.success.value: cls.os_template_image.progress = 100 else: cls.os_template_image.progress = 255 cls.os_template_image.update() elif _object == 'os_template_image': if action == 'delete': cls.os_template_image.id = cls.message['message'][ 'passback_parameters']['id'] cls.os_template_image.get() if state == ResponseState.success.value: cls.os_template_image.delete() else: pass else: pass
def make_instance_snapshot_backup(instance, error): LOG.info("Make instance backup for %s" % (instance)) snapshot = Snapshot() snapshot.start_at = datetime.datetime.now() snapshot.type = Snapshot.SNAPSHOPT snapshot.status = Snapshot.RUNNING snapshot.instance = instance snapshot.environment = instance.databaseinfra.environment from dbaas_nfsaas.models import HostAttr as Nfsaas_HostAttr nfsaas_hostattr = Nfsaas_HostAttr.objects.get(host=instance.hostname) snapshot.export_path = nfsaas_hostattr.nfsaas_path databases = Database.objects.filter(databaseinfra=instance.databaseinfra) if databases: snapshot.database_name = databases[0].name snapshot.save() databaseinfra = instance.databaseinfra driver = databaseinfra.get_driver() client = driver.get_client(instance) cloudstack_hostattr = Cloudstack_HostAttr.objects.get( host=instance.hostname) try: LOG.debug('Locking instance %s' % str(instance)) driver.lock_database(client) LOG.debug('Instance %s is locked' % str(instance)) if type(driver).__name__ == 'MySQL': mysql_binlog_save(client, instance, cloudstack_hostattr) nfs_snapshot = NfsaasProvider.create_snapshot( environment=databaseinfra.environment, plan=databaseinfra.plan, host=instance.hostname) if 'error' in nfs_snapshot: errormsg = nfs_snapshot['error'] error['errormsg'] = errormsg set_backup_error(databaseinfra, snapshot, errormsg) return False if 'id' in nfs_snapshot and 'snapshot' in nfs_snapshot: snapshot.snapshopt_id = nfs_snapshot['id'] snapshot.snapshot_name = nfs_snapshot['snapshot'] else: errormsg = 'There is no snapshot information' error['errormsg'] = errormsg set_backup_error(databaseinfra, snapshot, errormsg) return False except Exception, e: errormsg = "Error creating snapshot: %s" % (e) error['errormsg'] = errormsg set_backup_error(databaseinfra, snapshot, errormsg) return False
class EventProcessor(object): message = None log = Log() guest = Guest() guest_migrate_info = GuestMigrateInfo() disk = Disk() snapshot = Snapshot() snapshot_disk_mapping = SnapshotDiskMapping() os_template_image = OSTemplateImage() config = Config() config.id = 1 guest_cpu_memory = GuestCPUMemory() guest_traffic = GuestTraffic() guest_disk_io = GuestDiskIO() host_cpu_memory = HostCPUMemory() host_traffic = HostTraffic() host_disk_usage_io = HostDiskUsageIO() @classmethod def log_processor(cls): cls.log.set(type=cls.message['type'], timestamp=cls.message['timestamp'], host=cls.message['host'], message=cls.message['message'], full_message='' if cls.message['message'].__len__() < 255 else cls.message['message']) cls.log.create() @classmethod def guest_event_processor(cls): cls.guest.uuid = cls.message['message']['uuid'] cls.guest.get_by('uuid') cls.guest.node_id = cls.message['node_id'] last_status = cls.guest.status cls.guest.status = cls.message['type'] if cls.message['type'] == GuestState.update.value: # 更新事件不改变 Guest 的状态 cls.guest.status = last_status cls.guest.xml = cls.message['message']['xml'] elif cls.guest.status == GuestState.migrating.value: try: cls.guest_migrate_info.uuid = cls.guest.uuid cls.guest_migrate_info.get_by('uuid') cls.guest_migrate_info.type = cls.message['message'][ 'migrating_info']['type'] cls.guest_migrate_info.time_elapsed = cls.message['message'][ 'migrating_info']['time_elapsed'] cls.guest_migrate_info.time_remaining = cls.message['message'][ 'migrating_info']['time_remaining'] cls.guest_migrate_info.data_total = cls.message['message'][ 'migrating_info']['data_total'] cls.guest_migrate_info.data_processed = cls.message['message'][ 'migrating_info']['data_processed'] cls.guest_migrate_info.data_remaining = cls.message['message'][ 'migrating_info']['data_remaining'] cls.guest_migrate_info.mem_total = cls.message['message'][ 'migrating_info']['mem_total'] cls.guest_migrate_info.mem_processed = cls.message['message'][ 'migrating_info']['mem_processed'] cls.guest_migrate_info.mem_remaining = cls.message['message'][ 'migrating_info']['mem_remaining'] cls.guest_migrate_info.file_total = cls.message['message'][ 'migrating_info']['file_total'] cls.guest_migrate_info.file_processed = cls.message['message'][ 'migrating_info']['file_processed'] cls.guest_migrate_info.file_remaining = cls.message['message'][ 'migrating_info']['file_remaining'] cls.guest_migrate_info.update() except ji.PreviewingError as e: ret = json.loads(e.message) if ret['state']['code'] == '404': cls.guest_migrate_info.type = cls.message['message'][ 'migrating_info']['type'] cls.guest_migrate_info.time_elapsed = cls.message[ 'message']['migrating_info']['time_elapsed'] cls.guest_migrate_info.time_remaining = cls.message[ 'message']['migrating_info']['time_remaining'] cls.guest_migrate_info.data_total = cls.message['message'][ 'migrating_info']['data_total'] cls.guest_migrate_info.data_processed = cls.message[ 'message']['migrating_info']['data_processed'] cls.guest_migrate_info.data_remaining = cls.message[ 'message']['migrating_info']['data_remaining'] cls.guest_migrate_info.mem_total = cls.message['message'][ 'migrating_info']['mem_total'] cls.guest_migrate_info.mem_processed = cls.message[ 'message']['migrating_info']['mem_processed'] cls.guest_migrate_info.mem_remaining = cls.message[ 'message']['migrating_info']['mem_remaining'] cls.guest_migrate_info.file_total = cls.message['message'][ 'migrating_info']['file_total'] cls.guest_migrate_info.file_processed = cls.message[ 'message']['migrating_info']['file_processed'] cls.guest_migrate_info.file_remaining = cls.message[ 'message']['migrating_info']['file_remaining'] cls.guest_migrate_info.create() elif cls.guest.status == GuestState.creating.value: if cls.message['message']['progress'] <= cls.guest.progress: return cls.guest.progress = cls.message['message']['progress'] elif cls.guest.status == GuestState.snapshot_converting.value: cls.os_template_image.id = cls.message['message'][ 'os_template_image_id'] cls.os_template_image.get() if cls.message['message'][ 'progress'] <= cls.os_template_image.progress: return cls.os_template_image.progress = cls.message['message']['progress'] cls.os_template_image.update() return cls.guest.update() # 限定特殊情况下更新磁盘所属 Guest,避免迁移、创建时频繁被无意义的更新 if cls.guest.status in [ GuestState.running.value, GuestState.shutoff.value ]: cls.disk.update_by_filter({'node_id': cls.guest.node_id}, filter_str='guest_uuid:eq:' + cls.guest.uuid) @classmethod def host_event_processor(cls): key = cls.message['message']['node_id'] value = { 'hostname': cls.message['host'], 'cpu': cls.message['message']['cpu'], 'system_load': cls.message['message']['system_load'], 'memory': cls.message['message']['memory'], 'memory_available': cls.message['message']['memory_available'], 'interfaces': cls.message['message']['interfaces'], 'disks': cls.message['message']['disks'], 'boot_time': cls.message['message']['boot_time'], 'nonrandom': False, 'threads_status': cls.message['message']['threads_status'], 'timestamp': ji.Common.ts() } db.r.hset(app.config['hosts_info'], key=key, value=json.dumps(value, ensure_ascii=False)) @classmethod def response_processor(cls): _object = cls.message['message']['_object'] action = cls.message['message']['action'] uuid = cls.message['message']['uuid'] state = cls.message['type'] data = cls.message['message']['data'] node_id = cls.message['node_id'] if _object == 'guest': if action == 'create': if state == ResponseState.success.value: # 系统盘的 UUID 与其 Guest 的 UUID 相同 cls.disk.uuid = uuid cls.disk.get_by('uuid') cls.disk.guest_uuid = uuid cls.disk.state = DiskState.mounted.value # disk_info['virtual-size'] 的单位为Byte,需要除以 1024 的 3 次方,换算成单位为 GB 的值 cls.disk.size = data['disk_info']['virtual-size'] / (1024** 3) cls.disk.update() else: cls.guest.uuid = uuid cls.guest.get_by('uuid') cls.guest.status = GuestState.dirty.value cls.guest.update() elif action == 'migrate': pass elif action == 'delete': if state == ResponseState.success.value: cls.config.get() cls.guest.uuid = uuid cls.guest.get_by('uuid') if IP(cls.config.start_ip).int() <= IP( cls.guest.ip).int() <= IP(cls.config.end_ip).int(): if db.r.srem(app.config['ip_used_set'], cls.guest.ip): db.r.sadd(app.config['ip_available_set'], cls.guest.ip) if (cls.guest.vnc_port - cls.config.start_vnc_port) <= \ (IP(cls.config.end_ip).int() - IP(cls.config.start_ip).int()): if db.r.srem(app.config['vnc_port_used_set'], cls.guest.vnc_port): db.r.sadd(app.config['vnc_port_available_set'], cls.guest.vnc_port) cls.guest.delete() # TODO: 加入是否删除使用的数据磁盘开关,如果为True,则顺便删除使用的磁盘。否则解除该磁盘被使用的状态。 cls.disk.uuid = uuid cls.disk.get_by('uuid') cls.disk.delete() cls.disk.update_by_filter( { 'guest_uuid': '', 'sequence': -1, 'state': DiskState.idle.value }, filter_str='guest_uuid:eq:' + cls.guest.uuid) SSHKeyGuestMapping.delete_by_filter(filter_str=':'.join( ['guest_uuid', 'eq', cls.guest.uuid])) elif action == 'reset_password': if state == ResponseState.success.value: cls.guest.uuid = uuid cls.guest.get_by('uuid') cls.guest.password = cls.message['message'][ 'passback_parameters']['password'] cls.guest.update() elif action == 'attach_disk': cls.disk.uuid = cls.message['message']['passback_parameters'][ 'disk_uuid'] cls.disk.get_by('uuid') if state == ResponseState.success.value: cls.disk.guest_uuid = uuid cls.disk.sequence = cls.message['message'][ 'passback_parameters']['sequence'] cls.disk.state = DiskState.mounted.value cls.disk.update() elif action == 'detach_disk': cls.disk.uuid = cls.message['message']['passback_parameters'][ 'disk_uuid'] cls.disk.get_by('uuid') if state == ResponseState.success.value: cls.disk.guest_uuid = '' cls.disk.sequence = -1 cls.disk.state = DiskState.idle.value cls.disk.update() elif action == 'boot': if state == ResponseState.success.value: pass elif _object == 'disk': if action == 'create': cls.disk.uuid = uuid cls.disk.get_by('uuid') cls.disk.node_id = node_id if state == ResponseState.success.value: cls.disk.state = DiskState.idle.value else: cls.disk.state = DiskState.dirty.value cls.disk.update() elif action == 'resize': if state == ResponseState.success.value: cls.config.get() cls.disk.uuid = uuid cls.disk.get_by('uuid') cls.disk.size = cls.message['message'][ 'passback_parameters']['size'] cls.disk.quota(config=cls.config) cls.disk.update() elif action == 'delete': cls.disk.uuid = uuid cls.disk.get_by('uuid') cls.disk.delete() elif _object == 'snapshot': if action == 'create': cls.snapshot.id = cls.message['message'][ 'passback_parameters']['id'] cls.snapshot.get() if state == ResponseState.success.value: cls.snapshot.snapshot_id = data['snapshot_id'] cls.snapshot.parent_id = data['parent_id'] cls.snapshot.xml = data['xml'] cls.snapshot.progress = 100 cls.snapshot.update() disks, _ = Disk.get_by_filter(filter_str='guest_uuid:eq:' + cls.snapshot.guest_uuid) for disk in disks: cls.snapshot_disk_mapping.snapshot_id = cls.snapshot.snapshot_id cls.snapshot_disk_mapping.disk_uuid = disk['uuid'] cls.snapshot_disk_mapping.create() else: cls.snapshot.progress = 255 cls.snapshot.update() if action == 'delete': if state == ResponseState.success.value: cls.snapshot.id = cls.message['message'][ 'passback_parameters']['id'] cls.snapshot.get() # 更新子快照的 parent_id 为,当前快照的 parent_id。因为当前快照已被删除。 Snapshot.update_by_filter( {'parent_id': cls.snapshot.parent_id}, filter_str='parent_id:eq:' + cls.snapshot.snapshot_id) SnapshotDiskMapping.delete_by_filter(filter_str=':'.join( ['snapshot_id', 'eq', cls.snapshot.snapshot_id])) cls.snapshot.delete() else: pass if action == 'revert': # 不论恢复成功与否,都使快照恢复至正常状态。 cls.snapshot.id = cls.message['message'][ 'passback_parameters']['id'] cls.snapshot.get() cls.snapshot.progress = 100 cls.snapshot.update() if action == 'convert': cls.snapshot.snapshot_id = cls.message['message'][ 'passback_parameters']['id'] cls.snapshot.get_by('snapshot_id') cls.snapshot.progress = 100 cls.snapshot.update() cls.os_template_image.id = cls.message['message'][ 'passback_parameters']['os_template_image_id'] cls.os_template_image.get() if state == ResponseState.success.value: cls.os_template_image.progress = 100 else: cls.os_template_image.progress = 255 cls.os_template_image.update() elif _object == 'os_template_image': if action == 'delete': cls.os_template_image.id = cls.message['message'][ 'passback_parameters']['id'] cls.os_template_image.get() if state == ResponseState.success.value: cls.os_template_image.delete() else: pass else: pass @classmethod def guest_collection_performance_processor(cls): data_kind = cls.message['type'] timestamp = ji.Common.ts() timestamp -= (timestamp % 60) data = cls.message['message']['data'] if data_kind == GuestCollectionPerformanceDataKind.cpu_memory.value: for item in data: cls.guest_cpu_memory.guest_uuid = item['guest_uuid'] cls.guest_cpu_memory.cpu_load = item['cpu_load'] cls.guest_cpu_memory.memory_available = item[ 'memory_available'] cls.guest_cpu_memory.memory_unused = item['memory_unused'] cls.guest_cpu_memory.timestamp = timestamp cls.guest_cpu_memory.create() if data_kind == GuestCollectionPerformanceDataKind.traffic.value: for item in data: cls.guest_traffic.guest_uuid = item['guest_uuid'] cls.guest_traffic.name = item['name'] cls.guest_traffic.rx_bytes = item['rx_bytes'] cls.guest_traffic.rx_packets = item['rx_packets'] cls.guest_traffic.rx_errs = item['rx_errs'] cls.guest_traffic.rx_drop = item['rx_drop'] cls.guest_traffic.tx_bytes = item['tx_bytes'] cls.guest_traffic.tx_packets = item['tx_packets'] cls.guest_traffic.tx_errs = item['tx_errs'] cls.guest_traffic.tx_drop = item['tx_drop'] cls.guest_traffic.timestamp = timestamp cls.guest_traffic.create() if data_kind == GuestCollectionPerformanceDataKind.disk_io.value: for item in data: cls.guest_disk_io.disk_uuid = item['disk_uuid'] cls.guest_disk_io.rd_req = item['rd_req'] cls.guest_disk_io.rd_bytes = item['rd_bytes'] cls.guest_disk_io.wr_req = item['wr_req'] cls.guest_disk_io.wr_bytes = item['wr_bytes'] cls.guest_disk_io.timestamp = timestamp cls.guest_disk_io.create() else: pass @classmethod def host_collection_performance_processor(cls): data_kind = cls.message['type'] timestamp = ji.Common.ts() timestamp -= (timestamp % 60) data = cls.message['message']['data'] if data_kind == HostCollectionPerformanceDataKind.cpu_memory.value: cls.host_cpu_memory.node_id = data['node_id'] cls.host_cpu_memory.cpu_load = data['cpu_load'] cls.host_cpu_memory.memory_available = data['memory_available'] cls.host_cpu_memory.timestamp = timestamp cls.host_cpu_memory.create() if data_kind == HostCollectionPerformanceDataKind.traffic.value: for item in data: cls.host_traffic.node_id = item['node_id'] cls.host_traffic.name = item['name'] cls.host_traffic.rx_bytes = item['rx_bytes'] cls.host_traffic.rx_packets = item['rx_packets'] cls.host_traffic.rx_errs = item['rx_errs'] cls.host_traffic.rx_drop = item['rx_drop'] cls.host_traffic.tx_bytes = item['tx_bytes'] cls.host_traffic.tx_packets = item['tx_packets'] cls.host_traffic.tx_errs = item['tx_errs'] cls.host_traffic.tx_drop = item['tx_drop'] cls.host_traffic.timestamp = timestamp cls.host_traffic.create() if data_kind == HostCollectionPerformanceDataKind.disk_usage_io.value: for item in data: cls.host_disk_usage_io.node_id = item['node_id'] cls.host_disk_usage_io.mountpoint = item['mountpoint'] cls.host_disk_usage_io.used = item['used'] cls.host_disk_usage_io.rd_req = item['rd_req'] cls.host_disk_usage_io.rd_bytes = item['rd_bytes'] cls.host_disk_usage_io.wr_req = item['wr_req'] cls.host_disk_usage_io.wr_bytes = item['wr_bytes'] cls.host_disk_usage_io.timestamp = timestamp cls.host_disk_usage_io.create() else: pass @classmethod def launch(cls): logger.info(msg='Thread EventProcessor is launched.') while True: if Utils.exit_flag: msg = 'Thread EventProcessor say bye-bye' print msg logger.info(msg=msg) return try: report = db.r.lpop(app.config['upstream_queue']) if report is None: time.sleep(1) continue cls.message = json.loads(report) if cls.message['kind'] == EmitKind.log.value: cls.log_processor() elif cls.message['kind'] == EmitKind.guest_event.value: cls.guest_event_processor() elif cls.message['kind'] == EmitKind.host_event.value: cls.host_event_processor() elif cls.message['kind'] == EmitKind.response.value: cls.response_processor() elif cls.message[ 'kind'] == EmitKind.guest_collection_performance.value: cls.guest_collection_performance_processor() elif cls.message[ 'kind'] == EmitKind.host_collection_performance.value: cls.host_collection_performance_processor() else: pass except Exception as e: logger.error(traceback.format_exc()) time.sleep(1)