def sprint(sprint_ids):
ids = sprint_ids.split(',')
if request.method == 'GET':
sprints = []
snapshots = []
last_snapshots = []
issues = []
committed_issues = []
states = _get_states_dict()
for sprint_id in ids:
sprint = Sprint.query.filter(Sprint.id == sprint_id).first()
sprints.append(sprint)
sprint_snapshots = sprint.get_snapshots()
snapshots.extend(sprint_snapshots)
last_snapshots.append(sprint_snapshots[-1])
# Get all of the issues captured in the latest snapshot
issue_snapshots = IssueSnapshot.query.filter(IssueSnapshot.snapshot_id == sprint.last_snapshot.id)
issues.extend([(issue.data, issue.state, issue, issue.sprint_count) for issue in issue_snapshots])
committed_issues.extend([commitment.issue_id for commitment in sprint.commitments])
stats_for_all_issues = get_stats_for_snapshots(snapshots)
completed_story_points = stats_for_all_issues['completed'][-1]
stats = {
'all': stats_for_all_issues,
'committed': get_stats_for_snapshots(snapshots, committed=True),
'label_stats': _get_label_statistics(sprints, completed_story_points),
}
snapshot_ids = [snapshot.id for snapshot in last_snapshots]
issue_state_stats = {
'all': {state: Snapshot.get_points_for_states(snapshot_ids, ids, [state]) for state in states.keys()},
'committed': {state: Snapshot.get_points_for_states(snapshot_ids, ids, [state], True) for state in states.keys()}
}
context = {
'sprints': sprints,
'snapshots': snapshots,
'states': states,
'stats': stats,
'issue_state_stats': issue_state_stats,
'issues': sorted(issues, key=lambda x: x[1]),
'committed_issues': committed_issues
}
return render_template('sprint.html', **context)
elif request.method == 'PATCH':
if len(ids) > 1:
return Response('Can only update one sprint at a time'), httplib.BAD_REQUEST
sprint = Sprint.query.filter(Sprint.id == ids[0]).first()
for name, value in request.json.items():
if hasattr(sprint, name):
setattr(sprint, name, value)
db_session.commit()
return Response(''), httplib.NO_CONTENT
def __at__(self, index):
''' Get snapshot at specific index '''
key = Snapshot.to_key(index + 1)
if key in self.cache:
return self.cache.get(key)
else:
snapshot = Snapshot.by_id(index + 1)
if snapshot is not None:
self.cache.set(snapshot.key, snapshot.to_dict())
return snapshot.to_dict()
return None
def snapshot_issues(repo, milestone):
"""
Fetches all of the issues for the given sprint and stores them in a database
"""
sprint = get_or_create_sprint(milestone)
print "Processing {} ({})".format(sprint.name, milestone['number'])
if sprint.locked is True:
print "Skipping '{}', it's locked".format(sprint.name)
return
snapshot = Snapshot(sprint)
db_session.add(snapshot)
url = 'https://api.github.com/repos/{}/{}/issues?state=all&milestone={}'.format(
settings.ORG, repo, milestone['number'])
have_updates = False
while True:
issues = _auth_get_request(url)
for issue in issues.json():
is_updated = save_issue_snapshot(repo, issue, snapshot)
have_updates = have_updates or is_updated
next_page = _get_next_page(issues)
if next_page:
url = next_page
else:
break
if have_updates:
print "Have updates, committing snapshot"
db_session.commit()
else:
print "No updates, reverting snapshot"
db_session.rollback()
def test_submit(self):
volley = [("Working - Coding", "Office", {
'happiness': 10,
'stress': 2
}, {
'activity': 'Working',
'activity_sub': "Coding",
'place': "Office"
}),
("Working: Meeting", "Office", {
'happiness': 2,
'stress': 4
}, {
'activity': 'Working',
'activity_sub': "Meeting",
'place': "Office"
}),
("Running", "Track", {
'happiness': 10,
'stress': 1
}, {
'activity': 'Running',
'activity_sub': None,
'place': "Track"
})]
for v in volley:
activity, place, metrics, expected_vals = v
kwargs = {'activity': activity, 'place': place, 'metrics': metrics}
sn = Snapshot.Create(self.u, **kwargs)
sn.put()
for key, val in expected_vals.items():
self.assertEqual(getattr(sn, key), val)
for metric, val in metrics.items():
self.assertEqual(sn.get_data_value(metric), val)
def _load(self):
''' Moves snapshots from db into the cache '''
logging.info("Loading game history from database ...")
if Snapshot.by_id(1) is None:
self.__now__() # Take starting snapshot
self.epoch = Snapshot.by_id(1).created
try:
max_index = len(self)
start_index = 1 if len(self) <= 10 else max_index - 9
for index in range(start_index, max_index + 1):
snapshot = Snapshot.by_id(index)
if not snapshot.key in self.cache:
logging.info("Cached snapshot (%d of %d)" %
(snapshot.id, max_index))
self.cache.set(snapshot.key, snapshot.to_dict())
logging.info("History load complete.")
except KeyboardInterrupt:
logging.info("History load stopped by user.")
def import_data():
snapshot = Snapshot()
snapshot.snapshot = datetime.now()
snapshot.save()
print "Importing data from finviz"
r = requests.get('http://finviz.com/export.ashx?v=152', cookies={"screenerUrl": "screener.ashx?v=152&f=cap_smallover&ft=4", "customTable": "0,1,2,6,7,10,11,13,14,45,65"})
data = csv_to_dicts(r.text)
tickers = []
for row in data:
try:
stock = Stock()
stock.snapshot = snapshot
if row["Ticker"]:
stock.Ticker = row["Ticker"]
print stock.Ticker
tickers.append(stock.Ticker)
if "Importing " + row["Company"]:
stock.Company = row["Company"]
if row["Market Cap"]:
stock.MarketCap = row["Market Cap"]
if row["P/E"]:
stock.PE = row["P/E"]
if row["P/S"]:
stock.PS = row["P/S"]
if row["P/B"]:
stock.PB = row["P/B"]
if row["P/Free Cash Flow"]:
stock.PFreeCashFlow = row["P/Free Cash Flow"]
if row["Dividend Yield"]:
stock.DividendYield = row["Dividend Yield"][:-1]
if row["Performance (Half Year)"]:
stock.PerformanceHalfYear = row["Performance (Half Year)"][:-1]
if row["Price"]:
stock.Price = row["Price"]
stock.save()
except:
pdb.set_trace()
import_evebitda(snapshot)
import_buyback_yield(snapshot)
def _load(self):
''' Moves snapshots from db into the cache '''
logging.info("Loading game history from database ...")
if Snapshot.by_id(1) is None:
self.__now__() # Take starting snapshot
self.epoch = Snapshot.by_id(1).created
try:
max_index = len(self)
start_index = 1 if len(self) <= 10 else max_index - 9
for index in range(start_index, max_index + 1):
snapshot = Snapshot.by_id(index)
if not snapshot.key in self.cache:
logging.info("Cached snapshot (%d of %d)" % (
snapshot.id, max_index
))
self.cache.set(snapshot.key, snapshot.to_dict())
logging.info("History load complete.")
except KeyboardInterrupt:
logging.info("History load stopped by user.")
def get_photos_geo(lat, lon, max_num=1):
snapshot_list = []
url = 'https://api.flickr.com/services/rest/?method=flickr.photos.search'
querystring = {"api_key": FLICKR_API_KEY, "lat":
lat, "lon": lon, "radius": "0.1", "format": "rest"}
headers = {'cache-control': "no-cache", }
response = requests.request(
"GET", url, headers=headers, params=querystring)
xml = xmltodict.parse(response.content)
# convert xml to json
json_string = json.dumps(xml)
json_data = json.loads(json_string)
counter = 0
try:
photos = json_data['rsp']['photos']['photo']
except:
return []
for photo in photos:
try:
id = photo['@id']
farm = photo['@farm']
server = photo['@server']
secret = photo['@secret']
image_uri = construct_uri(id, farm, server, secret)
val = get_photo_metadata(id)
if val == None:
continue
tags, date, views = val
datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S")
s = Snapshot(
image_uri=image_uri, views=views, date=datetime_obj, tags=tags)
snapshot_list.append(s)
except Exception as e:
continue
counter = counter + 1
if counter == max_num:
break
return snapshot_list
def snapshot(ctx, board, cycle_time, done):
ctx.obj['board_id'] = board
ts = TrelloStats(ctx.obj)
Snapshot.create_table(fail_silently=True)
"""
Recording mode - Daily snapshots of a board for ongoing reporting:
-> trellis report --board=87hiudhw
--cycle-time
--spend
--revenue
--done=Done
"""
if cycle_time:
done_id = ts.get_list_id_from_name(done)
cards = ts.get_list_data(done_id)
ct = ts.cycle_time(cards)
print ct
# Create snapshot
Snapshot.create(board_id=board, done_id=done_id, cycle_time=ct)
def r_revert(snapshot_id):
args_rules = [
Rules.SNAPSHOT_ID.value
]
try:
ret = dict()
ret['state'] = ji.Common.exchange_state(20000)
ji.Check.previewing(args_rules, {'snapshot_id': snapshot_id})
snapshot = Snapshot()
guest = Guest()
snapshot.snapshot_id = snapshot_id
snapshot.get_by('snapshot_id')
snapshot.progress = 253
snapshot.update()
snapshot.get()
guest.uuid = snapshot.guest_uuid
guest.get_by('uuid')
message = {
'_object': 'snapshot',
'action': 'revert',
'uuid': guest.uuid,
'snapshot_id': snapshot.snapshot_id,
'node_id': guest.node_id,
'passback_parameters': {'id': snapshot.id}
}
Utils.emit_instruction(message=json.dumps(message, ensure_ascii=False))
ret['data'] = snapshot.__dict__
return ret
except ji.PreviewingError, e:
return json.loads(e.message)
def submit(self, d):
'''
Submit a snapshot. Assume snapshot is now
'''
params = tools.gets(self,
strings=['lat', 'lon', 'activity', 'place'],
json=['metrics'],
lists=['people'])
snap = Snapshot.Create(self.user, **params)
snap.put()
self.success = True
self.set_response({'snapshot': snap.json() if snap else None},
message="Snapshot submitted!",
debug=True)
def print_stats(sprint_name=None):
if sprint_name:
sprints = Sprint.query.filter(Sprint.name == sprint_name)
else:
sprints = Sprint.query.all()
for sprint in sprints:
print sprint.name
snapshot = Snapshot.get_most_recent_for_sprint(sprint)
if snapshot is None:
print " No stats exist"
else:
issues = snapshot.issues
for state in settings.ISSUE_STATES:
print " {}: {}".format(state['label'], _sum_points(issues, state['id']))
def print_stats(sprint_name=None):
if sprint_name:
sprints = Sprint.query.filter(Sprint.name == sprint_name)
else:
sprints = Sprint.query.all()
for sprint in sprints:
print sprint.name
snapshot = Snapshot.get_most_recent_for_sprint(sprint)
if snapshot is None:
print " No stats exist"
else:
issues = snapshot.issues
for state in settings.ISSUE_STATES:
print " {}: {}".format(state['label'], _sum_points(issues, state['id']))
def r_update(snapshot_id):
snapshot = Snapshot()
args_rules = [
Rules.SNAPSHOT_ID.value
]
if 'label' in request.json:
args_rules.append(
Rules.LABEL.value,
)
if args_rules.__len__() < 2:
ret = dict()
ret['state'] = ji.Common.exchange_state(20000)
return ret
request.json['snapshot_id'] = snapshot_id
try:
ji.Check.previewing(args_rules, request.json)
snapshot.snapshot_id = request.json.get('snapshot_id')
snapshot.get_by('snapshot_id')
snapshot.label = request.json.get('label', snapshot.label)
snapshot.update()
snapshot.get()
ret = dict()
ret['state'] = ji.Common.exchange_state(20000)
ret['data'] = snapshot.__dict__
return ret
except ji.PreviewingError, e:
return json.loads(e.message)
def create_snapshot(self, snapshot_name, before_copy=None):
snapshot = Snapshot(snapshot_name=snapshot_name,
project_name=self.config['project_name'])
self.db.session.add(snapshot)
self.db.session.flush()
for table_name in self.config['tracked_databases']:
if before_copy:
before_copy(table_name)
table = Table(table_name=table_name, snapshot=snapshot)
self.operations.copy_database(table_name,
table.get_table_name('master'))
self.db.session.add(table)
self.db.session.commit()
self.start_background_slave_copy(snapshot)
def __now__(self):
''' Returns snapshot object it as a dict '''
snapshot = Snapshot()
for team in Team.all():
snapshot_team = SnapshotTeam(
team_id=team.id,
money=team.money,
)
snapshot_team.game_levels = team.game_levels
snapshot_team.flags = team.flags
dbsession.add(snapshot_team)
dbsession.flush()
snapshot.teams.append(snapshot_team)
dbsession.add(snapshot)
dbsession.flush()
return snapshot
def __now__(self):
''' Returns snapshot object it as a dict '''
snapshot = Snapshot()
bot_manager = BotManager.Instance()
for team in Team.all():
snapshot_team = SnapshotTeam(team_id=team.id,
money=team.money,
bots=bot_manager.count_by_team(team))
snapshot_team.game_levels = team.game_levels
snapshot_team.flags = team.flags
dbsession.add(snapshot_team)
dbsession.flush()
snapshot.teams.append(snapshot_team)
dbsession.add(snapshot)
dbsession.flush()
return snapshot
def test_snapshot_calls(self):
# Create
snap = Snapshot.Create(self.u,
activity="Eating",
place="Restaurant",
people=["Elizabeth"],
metrics={'stress': 2})
snap.put()
self.assertEqual(snap.get_data_value('stress'), 2)
self.assertEqual(snap.activity, "Eating")
# List
response = self.get_json("/api/snapshot", {}, headers=self.api_headers)
snap = response.get('snapshots')[0]
print response
self.assertEqual(snap.get('activity'), "Eating")
def home():
return render_template("index.html", recent=Snapshot.get_recent())
def __contains__(self, index):
return True if Snapshot.by_id(index) is not None else False
def make_instance_snapshot_backup(instance, error, group, provider_class=VolumeProviderBase):
LOG.info("Make instance backup for {}".format(instance))
provider = provider_class(instance)
infra = instance.databaseinfra
database = infra.databases.first()
snapshot = Snapshot.create(instance, group, provider.volume)
snapshot_final_status = Snapshot.SUCCESS
locked = None
driver = infra.get_driver()
client = None
try:
client = driver.get_client(instance)
locked = lock_instance(driver, instance, client)
if not locked:
snapshot_final_status = Snapshot.WARNING
if 'MySQL' in type(driver).__name__:
mysql_binlog_save(client, instance)
response = provider.take_snapshot()
snapshot.done(response)
except Exception as e:
errormsg = "Error creating snapshot: {}".format(e)
error['errormsg'] = errormsg
set_backup_error(infra, snapshot, errormsg)
return snapshot
finally:
if locked:
unlock_instance(driver, instance, client)
output = {}
command = "du -sb /data/.snapshot/%s | awk '{print $1}'" % (
snapshot.snapshot_name
)
try:
exec_remote_command_host(instance.hostname, command, output)
size = int(output['stdout'][0])
snapshot.size = size
except Exception as e:
snapshot.size = 0
LOG.error("Error exec remote command {}".format(e))
backup_path = database.backup_path
if backup_path:
now = datetime.now()
target_path = "{}/{}/{}/{}/{}".format(
backup_path,
now.strftime("%Y_%m_%d"),
instance.hostname.hostname.split('.')[0],
now.strftime("%Y%m%d%H%M%S"),
infra.name
)
snapshot_path = "/data/.snapshot/{}/data/".format(
snapshot.snapshot_name
)
output = {}
command = """
if [ -d "{backup_path}" ]
then
rm -rf {backup_path}/20[0-9][0-9]_[0-1][0-9]_[0-3][0-9] &
mkdir -p {target_path}
cp -r {snapshot_path} {target_path} &
fi
""".format(backup_path=backup_path,
target_path=target_path,
snapshot_path=snapshot_path)
try:
exec_remote_command_host(instance.hostname, command, output)
except Exception as e:
LOG.error("Error exec remote command {}".format(e))
snapshot.status = snapshot_final_status
snapshot.end_at = datetime.now()
snapshot.save()
register_backup_dbmonitor(infra, snapshot)
return snapshot
def resetdb(ctx):
Snapshot.drop_table()
Snapshot.create_table()
click.echo('Snapshots table dropped.')
def make_instance_snapshot_backup(instance, error):
LOG.info("Make instance backup for %s" % (instance))
snapshot = Snapshot()
snapshot.start_at = datetime.datetime.now()
snapshot.type = Snapshot.SNAPSHOPT
snapshot.status = Snapshot.RUNNING
snapshot.instance = instance
snapshot.environment = instance.databaseinfra.environment
from dbaas_nfsaas.models import HostAttr as Nfsaas_HostAttr
nfsaas_hostattr = Nfsaas_HostAttr.objects.get(
host=instance.hostname, is_active=True)
snapshot.export_path = nfsaas_hostattr.nfsaas_path
databases = Database.objects.filter(databaseinfra=instance.databaseinfra)
if databases:
snapshot.database_name = databases[0].name
snapshot.save()
databaseinfra = instance.databaseinfra
driver = databaseinfra.get_driver()
client = driver.get_client(instance)
cloudstack_hostattr = Cloudstack_HostAttr.objects.get(
host=instance.hostname)
try:
LOG.debug('Locking instance %s' % str(instance))
driver.lock_database(client)
LOG.debug('Instance %s is locked' % str(instance))
if type(driver).__name__ == 'MySQL':
mysql_binlog_save(client, instance, cloudstack_hostattr)
nfs_snapshot = NfsaasProvider.create_snapshot(environment=databaseinfra.environment,
host=instance.hostname)
if 'error' in nfs_snapshot:
errormsg = nfs_snapshot['error']
error['errormsg'] = errormsg
set_backup_error(databaseinfra, snapshot, errormsg)
return False
if 'id' in nfs_snapshot and 'snapshot' in nfs_snapshot:
snapshot.snapshopt_id = nfs_snapshot['id']
snapshot.snapshot_name = nfs_snapshot['snapshot']
else:
errormsg = 'There is no snapshot information'
error['errormsg'] = errormsg
set_backup_error(databaseinfra, snapshot, errormsg)
return False
except Exception as e:
errormsg = "Error creating snapshot: %s" % (e)
error['errormsg'] = errormsg
set_backup_error(databaseinfra, snapshot, errormsg)
return False
finally:
LOG.debug('Unlocking instance %s' % str(instance))
driver.unlock_database(client)
LOG.debug('Instance %s is unlocked' % str(instance))
output = {}
command = "du -sb /data/.snapshot/%s | awk '{print $1}'" % (
snapshot.snapshot_name)
try:
exec_remote_command(server=instance.hostname.address,
username=cloudstack_hostattr.vm_user,
password=cloudstack_hostattr.vm_password,
command=command,
output=output)
size = int(output['stdout'][0])
snapshot.size = size
except Exception as e:
snapshot.size = 0
LOG.error("Error exec remote command %s" % (e))
backup_path = databases[0].backup_path
if backup_path:
infraname = databaseinfra.name
now = datetime.datetime.now()
target_path = "{backup_path}/{today_str}/{hostname}/{now_str}/{infraname}".format(
backup_path=backup_path,
today_str=now.strftime("%Y_%m_%d"),
hostname=instance.hostname.hostname.split('.')[0],
now_str=now.strftime("%Y%m%d%H%M%S"),
infraname=infraname)
snapshot_path = "/data/.snapshot/{}/data/".format(snapshot.snapshot_name)
output = {}
command = """
if [ -d "{backup_path}" ]
then
rm -rf {backup_path}/20[0-9][0-9]_[0-1][0-12]_[0-3][0-9]
mkdir -p {target_path}
cp -r {snapshot_path} {target_path} &
fi
""".format(backup_path=backup_path,
target_path=target_path,
snapshot_path=snapshot_path)
try:
exec_remote_command(server=instance.hostname.address,
username=cloudstack_hostattr.vm_user,
password=cloudstack_hostattr.vm_password,
command=command,
output=output)
except Exception as e:
LOG.error("Error exec remote command %s" % (e))
snapshot.status = Snapshot.SUCCESS
snapshot.end_at = datetime.datetime.now()
snapshot.save()
register_backup_dbmonitor(databaseinfra, snapshot)
return True
def make_instance_snapshot_backup(instance, error):
LOG.info("Make instance backup for %s" % (instance))
snapshot = Snapshot()
snapshot.start_at = datetime.datetime.now()
snapshot.type=Snapshot.SNAPSHOPT
snapshot.status=Snapshot.RUNNING
snapshot.instance = instance
snapshot.environment = instance.databaseinfra.environment
from dbaas_nfsaas.models import HostAttr as Nfsaas_HostAttr
nfsaas_hostattr = Nfsaas_HostAttr.objects.get(host=instance.hostname)
snapshot.export_path = nfsaas_hostattr.nfsaas_path
databases = Database.objects.filter(databaseinfra=instance.databaseinfra)
if databases:
snapshot.database_name = databases[0].name
snapshot.save()
databaseinfra = instance.databaseinfra
driver = databaseinfra.get_driver()
client = driver.get_client(instance)
try:
driver.lock_database(client)
nfs_snapshot = NfsaasProvider.create_snapshot(environment = databaseinfra.environment,
plan = databaseinfra.plan,
host = instance.hostname)
driver.unlock_database(client)
if 'error' in nfs_snapshot:
errormsg = nfs_snapshot['error']
error['errormsg'] = errormsg
set_backup_error(databaseinfra, snapshot, errormsg)
return False
if 'id' in nfs_snapshot and 'snapshot' in nfs_snapshot:
snapshot.snapshopt_id = nfs_snapshot['id']
snapshot.snapshot_name = nfs_snapshot['snapshot']
else:
errormsg = 'There is no snapshot information'
error['errormsg'] = errormsg
set_backup_error(databaseinfra, snapshot, errormsg)
return False
except Exception, e:
errormsg = "Error creating snapshot: %s" % (e)
error['errormsg'] = errormsg
set_backup_error(databaseinfra, snapshot, errormsg)
return False
def list(self, d):
limit = self.request.get_range('limit', default=500)
snapshots = Snapshot.Recent(self.user, limit=limit)
self.set_response({
'snapshots': [s.json() for s in snapshots if s]
}, success=True, debug=True)
def r_create():
args_rules = [
Rules.GUEST_UUID.value
]
if 'label' in request.json:
args_rules.append(
Rules.LABEL.value,
)
try:
ret = dict()
ret['state'] = ji.Common.exchange_state(20000)
ji.Check.previewing(args_rules, request.json)
snapshot = Snapshot()
guest = Guest()
guest.uuid = request.json.get('guest_uuid')
guest.get_by('uuid')
snapshot.label = request.json.get('label', '')
snapshot.status = guest.status
snapshot.guest_uuid = guest.uuid
snapshot.snapshot_id = '_'.join(['tmp', ji.Common.generate_random_code(length=8)])
snapshot.parent_id = '-'
snapshot.progress = 0
snapshot.create()
snapshot.get_by('snapshot_id')
message = {
'_object': 'snapshot',
'action': 'create',
'uuid': guest.uuid,
'node_id': guest.node_id,
'passback_parameters': {'id': snapshot.id}
}
Utils.emit_instruction(message=json.dumps(message, ensure_ascii=False))
ret['data'] = snapshot.__dict__
return ret
except ji.PreviewingError, e:
return json.loads(e.message)
def r_convert_to_os_template_image(snapshot_id, disk_uuid):
args_rules = [
Rules.SNAPSHOT_ID.value,
Rules.DISK_UUID.value,
Rules.LABEL.value
]
try:
ret = dict()
ret['state'] = ji.Common.exchange_state(20000)
ji.Check.previewing(args_rules, {'snapshot_id': snapshot_id, 'disk_uuid': disk_uuid,
'label': request.json.get('label')})
rows, _ = SnapshotDiskMapping.get_by_filter(filter_str=':'.join(['snapshot_id', 'eq', snapshot_id]))
disks_uuid = list()
for row in rows:
disks_uuid.append(row['disk_uuid'])
if disk_uuid not in disks_uuid:
ret['state'] = ji.Common.exchange_state(40401)
ret['state']['sub']['zh-cn'] = ''.join([ret['state']['sub']['zh-cn'], u': 未在快照: ',
snapshot_id, u' 中找到磁盘：', disk_uuid])
return ret
config = Config()
config.id = 1
config.get()
snapshot = Snapshot()
os_template_image = OSTemplateImage()
guest = Guest()
disk = Disk()
snapshot.snapshot_id = snapshot_id
snapshot.get_by('snapshot_id')
snapshot.progress = 252
guest.uuid = snapshot.guest_uuid
guest.get_by('uuid')
disk.uuid = disk_uuid
disk.get_by('uuid')
os_template_image.id = guest.os_template_image_id
os_template_image.get()
image_name = '_'.join([snapshot.snapshot_id, disk.uuid]) + '.' + disk.format
os_template_image.id = 0
os_template_image.label = request.json.get('label')
os_template_image.path = '/'.join([os.path.dirname(os_template_image.path), image_name])
os_template_image.kind = OSTemplateImageKind.custom.value
os_template_image.progress = 0
os_template_image.create_time = ji.Common.tus()
if os_template_image.exist_by('path'):
ret['state'] = ji.Common.exchange_state(40901)
ret['state']['sub']['zh-cn'] = ''.join([ret['state']['sub']['zh-cn'], ': ', os_template_image.path])
return ret
os_template_image.create()
os_template_image.get_by('path')
message = {
'_object': 'snapshot',
'action': 'convert',
'uuid': disk.guest_uuid,
'snapshot_id': snapshot.snapshot_id,
'storage_mode': config.storage_mode,
'dfs_volume': config.dfs_volume,
'node_id': disk.node_id,
'snapshot_path': disk.path,
'template_path': os_template_image.path,
'os_template_image_id': os_template_image.id,
'passback_parameters': {'id': snapshot.snapshot_id, 'os_template_image_id': os_template_image.id}
}
Utils.emit_instruction(message=json.dumps(message, ensure_ascii=False))
snapshot.update()
return ret
except ji.PreviewingError, e:
return json.loads(e.message)
def r_content_search():
ret = guest_base.content_search()
uuids = list()
for guest in ret['data']:
uuids.append(guest['uuid'])
rows, _ = SSHKeyGuestMapping.get_by_filter(filter_str=':'.join(['guest_uuid', 'in', ','.join(uuids)]))
guest_uuid_ssh_key_id_mapping = dict()
ssh_keys_id = list()
for row in rows:
if row['ssh_key_id'] not in ssh_keys_id:
ssh_keys_id.append(row['ssh_key_id'].__str__())
if row['guest_uuid'] not in guest_uuid_ssh_key_id_mapping:
guest_uuid_ssh_key_id_mapping[row['guest_uuid']] = list()
guest_uuid_ssh_key_id_mapping[row['guest_uuid']].append(row['ssh_key_id'])
rows, _ = SSHKey.get_by_filter(filter_str=':'.join(['id', 'in', ','.join(ssh_keys_id)]))
ssh_key_id_mapping = dict()
for row in rows:
row['url'] = url_for('v_ssh_keys.show')
ssh_key_id_mapping[row['id']] = row
rows, _ = Snapshot.get_by_filter(filter_str=':'.join(['guest_uuid', 'in', ','.join(uuids)]))
snapshots_guest_uuid_mapping = dict()
for row in rows:
guest_uuid = row['guest_uuid']
if guest_uuid not in snapshots_guest_uuid_mapping:
snapshots_guest_uuid_mapping[guest_uuid] = list()
snapshots_guest_uuid_mapping[guest_uuid].append(row)
for i, guest in enumerate(ret['data']):
guest_uuid = ret['data'][i]['uuid']
if 'ssh_keys' not in ret['data'][i]:
ret['data'][i]['ssh_keys'] = list()
if guest_uuid in guest_uuid_ssh_key_id_mapping:
for ssh_key_id in guest_uuid_ssh_key_id_mapping[guest_uuid]:
if ssh_key_id not in ssh_key_id_mapping:
continue
ret['data'][i]['ssh_keys'].append(ssh_key_id_mapping[ssh_key_id])
if 'snapshot' not in ret['data'][i]:
ret['data'][i]['snapshot'] = {
'creatable': True,
'mapping': list()
}
if guest_uuid in snapshots_guest_uuid_mapping:
ret['data'][i]['snapshot']['mapping'] = snapshots_guest_uuid_mapping[guest_uuid]
for snapshot in snapshots_guest_uuid_mapping[guest_uuid]:
if snapshot['progress'] == 100:
continue
else:
ret['data'][i]['snapshot']['creatable'] = False
return ret
def r_delete(snapshots_id):
args_rules = [
Rules.SNAPSHOTS_ID.value
]
try:
ji.Check.previewing(args_rules, {'snapshots_id': snapshots_id})
snapshot = Snapshot()
guest = Guest()
# 检测所指定的 快照 都存在
for snapshot_id in snapshots_id.split(','):
snapshot.snapshot_id = snapshot_id
snapshot.get_by('snapshot_id')
guest.uuid = snapshot.guest_uuid
guest.get_by('uuid')
# 执行删除操作
for snapshot_id in snapshots_id.split(','):
snapshot.snapshot_id = snapshot_id
snapshot.get_by('snapshot_id')
guest.uuid = snapshot.guest_uuid
guest.get_by('uuid')
message = {
'_object': 'snapshot',
'action': 'delete',
'uuid': snapshot.guest_uuid,
'snapshot_id': snapshot.snapshot_id,
'node_id': guest.node_id,
'passback_parameters': {'id': snapshot.id}
}
Utils.emit_instruction(message=json.dumps(message))
# 删除创建失败的 快照
if snapshot.progress == 255:
SnapshotDiskMapping.delete_by_filter(filter_str=':'.join(['snapshot_id', 'eq', snapshot.snapshot_id]))
snapshot.delete()
else:
snapshot.progress = 254
snapshot.update()
ret = dict()
ret['state'] = ji.Common.exchange_state(20000)
return ret
except ji.PreviewingError, e:
return json.loads(e.message)
def make_instance_snapshot_backup(instance, error):
LOG.info("Make instance backup for %s" % (instance))
snapshot = Snapshot()
snapshot.start_at = datetime.datetime.now()
snapshot.type = Snapshot.SNAPSHOPT
snapshot.status = Snapshot.RUNNING
snapshot.instance = instance
snapshot.environment = instance.databaseinfra.environment
from dbaas_nfsaas.models import HostAttr as Nfsaas_HostAttr
nfsaas_hostattr = Nfsaas_HostAttr.objects.get(
host=instance.hostname, is_active=True)
snapshot.export_path = nfsaas_hostattr.nfsaas_path
databases = Database.objects.filter(databaseinfra=instance.databaseinfra)
if databases:
snapshot.database_name = databases[0].name
snapshot.save()
databaseinfra = instance.databaseinfra
driver = databaseinfra.get_driver()
client = driver.get_client(instance)
cloudstack_hostattr = Cloudstack_HostAttr.objects.get(
host=instance.hostname)
try:
LOG.debug('Locking instance %s' % str(instance))
driver.lock_database(client)
LOG.debug('Instance %s is locked' % str(instance))
if type(driver).__name__ == 'MySQL':
mysql_binlog_save(client, instance, cloudstack_hostattr)
nfs_snapshot = NfsaasProvider.create_snapshot(environment=databaseinfra.environment,
host=instance.hostname)
if 'error' in nfs_snapshot:
errormsg = nfs_snapshot['error']
error['errormsg'] = errormsg
set_backup_error(databaseinfra, snapshot, errormsg)
return False
if 'id' in nfs_snapshot and 'snapshot' in nfs_snapshot:
snapshot.snapshopt_id = nfs_snapshot['id']
snapshot.snapshot_name = nfs_snapshot['snapshot']
else:
errormsg = 'There is no snapshot information'
error['errormsg'] = errormsg
set_backup_error(databaseinfra, snapshot, errormsg)
return False
except Exception, e:
errormsg = "Error creating snapshot: %s" % (e)
error['errormsg'] = errormsg
set_backup_error(databaseinfra, snapshot, errormsg)
return False
def snapshot_result(guid):
snapshot = db.session.query(Snapshot).get(guid)
return render_template("snapshot_result.html", snapshot=snapshot, recent=Snapshot.get_recent())
def make_instance_snapshot_backup(instance,
error,
group,
provider_class=VolumeProviderBase):
LOG.info("Make instance backup for {}".format(instance))
provider = provider_class(instance)
infra = instance.databaseinfra
database = infra.databases.first()
snapshot = Snapshot.create(instance, group, provider.volume)
snapshot_final_status = Snapshot.SUCCESS
locked = None
driver = infra.get_driver()
client = None
try:
client = driver.get_client(instance)
locked = lock_instance(driver, instance, client)
if not locked:
snapshot_final_status = Snapshot.WARNING
if 'MySQL' in type(driver).__name__:
mysql_binlog_save(client, instance)
response = provider.take_snapshot()
snapshot.done(response)
except Exception as e:
errormsg = "Error creating snapshot: {}".format(e)
error['errormsg'] = errormsg
set_backup_error(infra, snapshot, errormsg)
return snapshot
finally:
unlock_instance(driver, instance, client)
output = {}
command = "du -sb /data/.snapshot/%s | awk '{print $1}'" % (
snapshot.snapshot_name)
try:
exec_remote_command_host(instance.hostname, command, output)
size = int(output['stdout'][0])
snapshot.size = size
except Exception as e:
snapshot.size = 0
LOG.error("Error exec remote command {}".format(e))
backup_path = database.backup_path
if backup_path:
now = datetime.now()
target_path = "{}/{}/{}/{}/{}".format(
backup_path, now.strftime("%Y_%m_%d"),
instance.hostname.hostname.split('.')[0],
now.strftime("%Y%m%d%H%M%S"), infra.name)
snapshot_path = "/data/.snapshot/{}/data/".format(
snapshot.snapshot_name)
output = {}
command = """
if [ -d "{backup_path}" ]
then
rm -rf {backup_path}/20[0-9][0-9]_[0-1][0-9]_[0-3][0-9] &
mkdir -p {target_path}
cp -r {snapshot_path} {target_path} &
fi
""".format(backup_path=backup_path,
target_path=target_path,
snapshot_path=snapshot_path)
try:
exec_remote_command_host(instance.hostname, command, output)
except Exception as e:
LOG.error("Error exec remote command {}".format(e))
snapshot.status = snapshot_final_status
snapshot.end_at = datetime.now()
snapshot.save()
register_backup_dbmonitor(infra, snapshot)
return snapshot
def make_instance_snapshot_backup(instance, error):
LOG.info("Make instance backup for %s" % (instance))
snapshot = Snapshot()
snapshot.start_at = datetime.datetime.now()
snapshot.type = Snapshot.SNAPSHOPT
snapshot.status = Snapshot.RUNNING
snapshot.instance = instance
snapshot.environment = instance.databaseinfra.environment
from dbaas_nfsaas.models import HostAttr as Nfsaas_HostAttr
nfsaas_hostattr = Nfsaas_HostAttr.objects.get(
host=instance.hostname, is_active=True
)
snapshot.export_path = nfsaas_hostattr.nfsaas_path
databases = Database.objects.filter(databaseinfra=instance.databaseinfra)
if databases:
snapshot.database_name = databases[0].name
snapshot.save()
snapshot_final_status = Snapshot.SUCCESS
try:
databaseinfra = instance.databaseinfra
driver = databaseinfra.get_driver()
client = driver.get_client(instance)
cloudstack_hostattr = Cloudstack_HostAttr.objects.get(
host=instance.hostname
)
locked = lock_instance(driver, instance, client)
if not locked:
snapshot_final_status = Snapshot.WARNING
if type(driver).__name__ == 'MySQL':
mysql_binlog_save(client, instance, cloudstack_hostattr)
nfs_snapshot = create_snapshot(
environment=databaseinfra.environment, host=instance.hostname
)
if 'id' in nfs_snapshot and 'name' in nfs_snapshot:
snapshot.snapshopt_id = nfs_snapshot['id']
snapshot.snapshot_name = nfs_snapshot['name']
else:
errormsg = 'There is no snapshot information'
error['errormsg'] = errormsg
set_backup_error(databaseinfra, snapshot, errormsg)
return snapshot
except Exception as e:
errormsg = "Error creating snapshot: %s" % (e)
error['errormsg'] = errormsg
set_backup_error(databaseinfra, snapshot, errormsg)
return snapshot
finally:
if locked:
unlock_instance(driver, instance, client)
output = {}
command = "du -sb /data/.snapshot/%s | awk '{print $1}'" % (
snapshot.snapshot_name)
try:
exec_remote_command(server=instance.hostname.address,
username=cloudstack_hostattr.vm_user,
password=cloudstack_hostattr.vm_password,
command=command,
output=output)
size = int(output['stdout'][0])
snapshot.size = size
except Exception as e:
snapshot.size = 0
LOG.error("Error exec remote command %s" % (e))
backup_path = databases[0].backup_path
if backup_path:
infraname = databaseinfra.name
now = datetime.datetime.now()
target_path = "{backup_path}/{today_str}/{hostname}/{now_str}/{infraname}".format(
backup_path=backup_path,
today_str=now.strftime("%Y_%m_%d"),
hostname=instance.hostname.hostname.split('.')[0],
now_str=now.strftime("%Y%m%d%H%M%S"),
infraname=infraname)
snapshot_path = "/data/.snapshot/{}/data/".format(snapshot.snapshot_name)
output = {}
command = """
if [ -d "{backup_path}" ]
then
rm -rf {backup_path}/20[0-9][0-9]_[0-1][0-9]_[0-3][0-9] &
mkdir -p {target_path}
cp -r {snapshot_path} {target_path} &
fi
""".format(backup_path=backup_path,
target_path=target_path,
snapshot_path=snapshot_path)
try:
exec_remote_command(server=instance.hostname.address,
username=cloudstack_hostattr.vm_user,
password=cloudstack_hostattr.vm_password,
command=command,
output=output)
except Exception as e:
LOG.error("Error exec remote command %s" % (e))
snapshot.status = snapshot_final_status
snapshot.end_at = datetime.datetime.now()
snapshot.save()
register_backup_dbmonitor(databaseinfra, snapshot)
return snapshot
def make_instance_snapshot_backup(instance, error, group,
provider_class=VolumeProviderSnapshot,
target_volume=None,
current_hour=None):
LOG.info("Make instance backup for {}".format(instance))
provider = provider_class(instance)
infra = instance.databaseinfra
database = infra.databases.first()
snapshot = Snapshot.create(
instance, group,
target_volume or provider.volume,
environment=provider.environment
)
snapshot_final_status = Snapshot.SUCCESS
locked = None
driver = infra.get_driver()
client = None
try:
client = driver.get_client(instance)
locked = lock_instance(driver, instance, client)
if not locked:
snapshot_final_status = Snapshot.WARNING
if 'MySQL' in type(driver).__name__:
mysql_binlog_save(client, instance)
current_time = datetime.now()
has_snapshot = Snapshot.objects.filter(
status=Snapshot.WARNING,
instance=instance,
end_at__year=current_time.year,
end_at__month=current_time.month,
end_at__day=current_time.day
)
backup_hour_list = Configuration.get_by_name_as_list(
'make_database_backup_hour'
)
if (snapshot_final_status == Snapshot.WARNING and has_snapshot):
if str(current_hour) in backup_hour_list:
raise Exception(
"Backup with WARNING already created today."
)
else:
response = provider.take_snapshot()
snapshot.done(response)
snapshot.save()
except Exception as e:
errormsg = "Error creating snapshot: {}".format(e)
error['errormsg'] = errormsg
set_backup_error(infra, snapshot, errormsg)
return snapshot
finally:
unlock_instance(driver, instance, client)
if not snapshot.size:
command = "du -sb /data/.snapshot/%s | awk '{print $1}'" % (
snapshot.snapshot_name
)
try:
output = instance.hostname.ssh.run_script(command)
size = int(output['stdout'][0])
snapshot.size = size
except Exception as e:
snapshot.size = 0
LOG.error("Error exec remote command {}".format(e))
backup_path = database.backup_path
if backup_path:
now = datetime.now()
target_path = "{}/{}/{}/{}/{}".format(
backup_path,
now.strftime("%Y_%m_%d"),
instance.hostname.hostname.split('.')[0],
now.strftime("%Y%m%d%H%M%S"),
infra.name
)
snapshot_path = "/data/.snapshot/{}/data/".format(
snapshot.snapshot_name
)
command = """
if [ -d "{backup_path}" ]
then
rm -rf {backup_path}/20[0-9][0-9]_[0-1][0-9]_[0-3][0-9] &
mkdir -p {target_path}
cp -r {snapshot_path} {target_path} &
fi
""".format(backup_path=backup_path,
target_path=target_path,
snapshot_path=snapshot_path)
try:
instance.hostname.ssh.run_script(command)
except Exception as e:
LOG.error("Error exec remote command {}".format(e))
snapshot.status = snapshot_final_status
snapshot.end_at = datetime.now()
snapshot.save()
register_backup_dbmonitor(infra, snapshot)
return snapshot
def response_processor(cls):
_object = cls.message['message']['_object']
action = cls.message['message']['action']
uuid = cls.message['message']['uuid']
state = cls.message['type']
data = cls.message['message']['data']
node_id = cls.message['node_id']
if _object == 'guest':
if action == 'create':
if state == ResponseState.success.value:
# 系统盘的 UUID 与其 Guest 的 UUID 相同
cls.disk.uuid = uuid
cls.disk.get_by('uuid')
cls.disk.guest_uuid = uuid
cls.disk.state = DiskState.mounted.value
# disk_info['virtual-size'] 的单位为Byte，需要除以 1024 的 3 次方，换算成单位为 GB 的值
cls.disk.size = data['disk_info']['virtual-size'] / (1024**
3)
cls.disk.update()
else:
cls.guest.uuid = uuid
cls.guest.get_by('uuid')
cls.guest.status = GuestState.dirty.value
cls.guest.update()
elif action == 'migrate':
pass
elif action == 'delete':
if state == ResponseState.success.value:
cls.config.get()
cls.guest.uuid = uuid
cls.guest.get_by('uuid')
if IP(cls.config.start_ip).int() <= IP(
cls.guest.ip).int() <= IP(cls.config.end_ip).int():
if db.r.srem(app.config['ip_used_set'], cls.guest.ip):
db.r.sadd(app.config['ip_available_set'],
cls.guest.ip)
if (cls.guest.vnc_port - cls.config.start_vnc_port) <= \
(IP(cls.config.end_ip).int() - IP(cls.config.start_ip).int()):
if db.r.srem(app.config['vnc_port_used_set'],
cls.guest.vnc_port):
db.r.sadd(app.config['vnc_port_available_set'],
cls.guest.vnc_port)
cls.guest.delete()
# TODO: 加入是否删除使用的数据磁盘开关，如果为True，则顺便删除使用的磁盘。否则解除该磁盘被使用的状态。
cls.disk.uuid = uuid
cls.disk.get_by('uuid')
cls.disk.delete()
cls.disk.update_by_filter(
{
'guest_uuid': '',
'sequence': -1,
'state': DiskState.idle.value
},
filter_str='guest_uuid:eq:' + cls.guest.uuid)
SSHKeyGuestMapping.delete_by_filter(filter_str=':'.join(
['guest_uuid', 'eq', cls.guest.uuid]))
elif action == 'reset_password':
if state == ResponseState.success.value:
cls.guest.uuid = uuid
cls.guest.get_by('uuid')
cls.guest.password = cls.message['message'][
'passback_parameters']['password']
cls.guest.update()
elif action == 'attach_disk':
cls.disk.uuid = cls.message['message']['passback_parameters'][
'disk_uuid']
cls.disk.get_by('uuid')
if state == ResponseState.success.value:
cls.disk.guest_uuid = uuid
cls.disk.sequence = cls.message['message'][
'passback_parameters']['sequence']
cls.disk.state = DiskState.mounted.value
cls.disk.update()
elif action == 'detach_disk':
cls.disk.uuid = cls.message['message']['passback_parameters'][
'disk_uuid']
cls.disk.get_by('uuid')
if state == ResponseState.success.value:
cls.disk.guest_uuid = ''
cls.disk.sequence = -1
cls.disk.state = DiskState.idle.value
cls.disk.update()
elif action == 'boot':
if state == ResponseState.success.value:
pass
elif _object == 'disk':
if action == 'create':
cls.disk.uuid = uuid
cls.disk.get_by('uuid')
cls.disk.node_id = node_id
if state == ResponseState.success.value:
cls.disk.state = DiskState.idle.value
else:
cls.disk.state = DiskState.dirty.value
cls.disk.update()
elif action == 'resize':
if state == ResponseState.success.value:
cls.config.get()
cls.disk.uuid = uuid
cls.disk.get_by('uuid')
cls.disk.size = cls.message['message'][
'passback_parameters']['size']
cls.disk.quota(config=cls.config)
cls.disk.update()
elif action == 'delete':
cls.disk.uuid = uuid
cls.disk.get_by('uuid')
cls.disk.delete()
elif _object == 'snapshot':
if action == 'create':
cls.snapshot.id = cls.message['message'][
'passback_parameters']['id']
cls.snapshot.get()
if state == ResponseState.success.value:
cls.snapshot.snapshot_id = data['snapshot_id']
cls.snapshot.parent_id = data['parent_id']
cls.snapshot.xml = data['xml']
cls.snapshot.progress = 100
cls.snapshot.update()
disks, _ = Disk.get_by_filter(filter_str='guest_uuid:eq:' +
cls.snapshot.guest_uuid)
for disk in disks:
cls.snapshot_disk_mapping.snapshot_id = cls.snapshot.snapshot_id
cls.snapshot_disk_mapping.disk_uuid = disk['uuid']
cls.snapshot_disk_mapping.create()
else:
cls.snapshot.progress = 255
cls.snapshot.update()
if action == 'delete':
if state == ResponseState.success.value:
cls.snapshot.id = cls.message['message'][
'passback_parameters']['id']
cls.snapshot.get()
# 更新子快照的 parent_id 为，当前快照的 parent_id。因为当前快照已被删除。
Snapshot.update_by_filter(
{'parent_id': cls.snapshot.parent_id},
filter_str='parent_id:eq:' + cls.snapshot.snapshot_id)
SnapshotDiskMapping.delete_by_filter(filter_str=':'.join(
['snapshot_id', 'eq', cls.snapshot.snapshot_id]))
cls.snapshot.delete()
else:
pass
if action == 'revert':
# 不论恢复成功与否，都使快照恢复至正常状态。
cls.snapshot.id = cls.message['message'][
'passback_parameters']['id']
cls.snapshot.get()
cls.snapshot.progress = 100
cls.snapshot.update()
if action == 'convert':
cls.snapshot.snapshot_id = cls.message['message'][
'passback_parameters']['id']
cls.snapshot.get_by('snapshot_id')
cls.snapshot.progress = 100
cls.snapshot.update()
cls.os_template_image.id = cls.message['message'][
'passback_parameters']['os_template_image_id']
cls.os_template_image.get()
if state == ResponseState.success.value:
cls.os_template_image.progress = 100
else:
cls.os_template_image.progress = 255
cls.os_template_image.update()
elif _object == 'os_template_image':
if action == 'delete':
cls.os_template_image.id = cls.message['message'][
'passback_parameters']['id']
cls.os_template_image.get()
if state == ResponseState.success.value:
cls.os_template_image.delete()
else:
pass
else:
pass
def make_instance_snapshot_backup(instance, error):
LOG.info("Make instance backup for %s" % (instance))
snapshot = Snapshot()
snapshot.start_at = datetime.datetime.now()
snapshot.type = Snapshot.SNAPSHOPT
snapshot.status = Snapshot.RUNNING
snapshot.instance = instance
snapshot.environment = instance.databaseinfra.environment
from dbaas_nfsaas.models import HostAttr as Nfsaas_HostAttr
nfsaas_hostattr = Nfsaas_HostAttr.objects.get(host=instance.hostname)
snapshot.export_path = nfsaas_hostattr.nfsaas_path
databases = Database.objects.filter(databaseinfra=instance.databaseinfra)
if databases:
snapshot.database_name = databases[0].name
snapshot.save()
databaseinfra = instance.databaseinfra
driver = databaseinfra.get_driver()
client = driver.get_client(instance)
cloudstack_hostattr = Cloudstack_HostAttr.objects.get(
host=instance.hostname)
try:
LOG.debug('Locking instance %s' % str(instance))
driver.lock_database(client)
LOG.debug('Instance %s is locked' % str(instance))
if type(driver).__name__ == 'MySQL':
mysql_binlog_save(client, instance, cloudstack_hostattr)
nfs_snapshot = NfsaasProvider.create_snapshot(
environment=databaseinfra.environment,
plan=databaseinfra.plan,
host=instance.hostname)
if 'error' in nfs_snapshot:
errormsg = nfs_snapshot['error']
error['errormsg'] = errormsg
set_backup_error(databaseinfra, snapshot, errormsg)
return False
if 'id' in nfs_snapshot and 'snapshot' in nfs_snapshot:
snapshot.snapshopt_id = nfs_snapshot['id']
snapshot.snapshot_name = nfs_snapshot['snapshot']
else:
errormsg = 'There is no snapshot information'
error['errormsg'] = errormsg
set_backup_error(databaseinfra, snapshot, errormsg)
return False
except Exception, e:
errormsg = "Error creating snapshot: %s" % (e)
error['errormsg'] = errormsg
set_backup_error(databaseinfra, snapshot, errormsg)
return False
class EventProcessor(object):
message = None
log = Log()
guest = Guest()
guest_migrate_info = GuestMigrateInfo()
disk = Disk()
snapshot = Snapshot()
snapshot_disk_mapping = SnapshotDiskMapping()
os_template_image = OSTemplateImage()
config = Config()
config.id = 1
guest_cpu_memory = GuestCPUMemory()
guest_traffic = GuestTraffic()
guest_disk_io = GuestDiskIO()
host_cpu_memory = HostCPUMemory()
host_traffic = HostTraffic()
host_disk_usage_io = HostDiskUsageIO()
@classmethod
def log_processor(cls):
cls.log.set(type=cls.message['type'],
timestamp=cls.message['timestamp'],
host=cls.message['host'],
message=cls.message['message'],
full_message='' if cls.message['message'].__len__() < 255
else cls.message['message'])
cls.log.create()
@classmethod
def guest_event_processor(cls):
cls.guest.uuid = cls.message['message']['uuid']
cls.guest.get_by('uuid')
cls.guest.node_id = cls.message['node_id']
last_status = cls.guest.status
cls.guest.status = cls.message['type']
if cls.message['type'] == GuestState.update.value:
# 更新事件不改变 Guest 的状态
cls.guest.status = last_status
cls.guest.xml = cls.message['message']['xml']
elif cls.guest.status == GuestState.migrating.value:
try:
cls.guest_migrate_info.uuid = cls.guest.uuid
cls.guest_migrate_info.get_by('uuid')
cls.guest_migrate_info.type = cls.message['message'][
'migrating_info']['type']
cls.guest_migrate_info.time_elapsed = cls.message['message'][
'migrating_info']['time_elapsed']
cls.guest_migrate_info.time_remaining = cls.message['message'][
'migrating_info']['time_remaining']
cls.guest_migrate_info.data_total = cls.message['message'][
'migrating_info']['data_total']
cls.guest_migrate_info.data_processed = cls.message['message'][
'migrating_info']['data_processed']
cls.guest_migrate_info.data_remaining = cls.message['message'][
'migrating_info']['data_remaining']
cls.guest_migrate_info.mem_total = cls.message['message'][
'migrating_info']['mem_total']
cls.guest_migrate_info.mem_processed = cls.message['message'][
'migrating_info']['mem_processed']
cls.guest_migrate_info.mem_remaining = cls.message['message'][
'migrating_info']['mem_remaining']
cls.guest_migrate_info.file_total = cls.message['message'][
'migrating_info']['file_total']
cls.guest_migrate_info.file_processed = cls.message['message'][
'migrating_info']['file_processed']
cls.guest_migrate_info.file_remaining = cls.message['message'][
'migrating_info']['file_remaining']
cls.guest_migrate_info.update()
except ji.PreviewingError as e:
ret = json.loads(e.message)
if ret['state']['code'] == '404':
cls.guest_migrate_info.type = cls.message['message'][
'migrating_info']['type']
cls.guest_migrate_info.time_elapsed = cls.message[
'message']['migrating_info']['time_elapsed']
cls.guest_migrate_info.time_remaining = cls.message[
'message']['migrating_info']['time_remaining']
cls.guest_migrate_info.data_total = cls.message['message'][
'migrating_info']['data_total']
cls.guest_migrate_info.data_processed = cls.message[
'message']['migrating_info']['data_processed']
cls.guest_migrate_info.data_remaining = cls.message[
'message']['migrating_info']['data_remaining']
cls.guest_migrate_info.mem_total = cls.message['message'][
'migrating_info']['mem_total']
cls.guest_migrate_info.mem_processed = cls.message[
'message']['migrating_info']['mem_processed']
cls.guest_migrate_info.mem_remaining = cls.message[
'message']['migrating_info']['mem_remaining']
cls.guest_migrate_info.file_total = cls.message['message'][
'migrating_info']['file_total']
cls.guest_migrate_info.file_processed = cls.message[
'message']['migrating_info']['file_processed']
cls.guest_migrate_info.file_remaining = cls.message[
'message']['migrating_info']['file_remaining']
cls.guest_migrate_info.create()
elif cls.guest.status == GuestState.creating.value:
if cls.message['message']['progress'] <= cls.guest.progress:
return
cls.guest.progress = cls.message['message']['progress']
elif cls.guest.status == GuestState.snapshot_converting.value:
cls.os_template_image.id = cls.message['message'][
'os_template_image_id']
cls.os_template_image.get()
if cls.message['message'][
'progress'] <= cls.os_template_image.progress:
return
cls.os_template_image.progress = cls.message['message']['progress']
cls.os_template_image.update()
return
cls.guest.update()
# 限定特殊情况下更新磁盘所属 Guest，避免迁移、创建时频繁被无意义的更新
if cls.guest.status in [
GuestState.running.value, GuestState.shutoff.value
]:
cls.disk.update_by_filter({'node_id': cls.guest.node_id},
filter_str='guest_uuid:eq:' +
cls.guest.uuid)
@classmethod
def host_event_processor(cls):
key = cls.message['message']['node_id']
value = {
'hostname': cls.message['host'],
'cpu': cls.message['message']['cpu'],
'system_load': cls.message['message']['system_load'],
'memory': cls.message['message']['memory'],
'memory_available': cls.message['message']['memory_available'],
'interfaces': cls.message['message']['interfaces'],
'disks': cls.message['message']['disks'],
'boot_time': cls.message['message']['boot_time'],
'nonrandom': False,
'threads_status': cls.message['message']['threads_status'],
'timestamp': ji.Common.ts()
}
db.r.hset(app.config['hosts_info'],
key=key,
value=json.dumps(value, ensure_ascii=False))
@classmethod
def response_processor(cls):
_object = cls.message['message']['_object']
action = cls.message['message']['action']
uuid = cls.message['message']['uuid']
state = cls.message['type']
data = cls.message['message']['data']
node_id = cls.message['node_id']
if _object == 'guest':
if action == 'create':
if state == ResponseState.success.value:
# 系统盘的 UUID 与其 Guest 的 UUID 相同
cls.disk.uuid = uuid
cls.disk.get_by('uuid')
cls.disk.guest_uuid = uuid
cls.disk.state = DiskState.mounted.value
# disk_info['virtual-size'] 的单位为Byte，需要除以 1024 的 3 次方，换算成单位为 GB 的值
cls.disk.size = data['disk_info']['virtual-size'] / (1024**
3)
cls.disk.update()
else:
cls.guest.uuid = uuid
cls.guest.get_by('uuid')
cls.guest.status = GuestState.dirty.value
cls.guest.update()
elif action == 'migrate':
pass
elif action == 'delete':
if state == ResponseState.success.value:
cls.config.get()
cls.guest.uuid = uuid
cls.guest.get_by('uuid')
if IP(cls.config.start_ip).int() <= IP(
cls.guest.ip).int() <= IP(cls.config.end_ip).int():
if db.r.srem(app.config['ip_used_set'], cls.guest.ip):
db.r.sadd(app.config['ip_available_set'],
cls.guest.ip)
if (cls.guest.vnc_port - cls.config.start_vnc_port) <= \
(IP(cls.config.end_ip).int() - IP(cls.config.start_ip).int()):
if db.r.srem(app.config['vnc_port_used_set'],
cls.guest.vnc_port):
db.r.sadd(app.config['vnc_port_available_set'],
cls.guest.vnc_port)
cls.guest.delete()
# TODO: 加入是否删除使用的数据磁盘开关，如果为True，则顺便删除使用的磁盘。否则解除该磁盘被使用的状态。
cls.disk.uuid = uuid
cls.disk.get_by('uuid')
cls.disk.delete()
cls.disk.update_by_filter(
{
'guest_uuid': '',
'sequence': -1,
'state': DiskState.idle.value
},
filter_str='guest_uuid:eq:' + cls.guest.uuid)
SSHKeyGuestMapping.delete_by_filter(filter_str=':'.join(
['guest_uuid', 'eq', cls.guest.uuid]))
elif action == 'reset_password':
if state == ResponseState.success.value:
cls.guest.uuid = uuid
cls.guest.get_by('uuid')
cls.guest.password = cls.message['message'][
'passback_parameters']['password']
cls.guest.update()
elif action == 'attach_disk':
cls.disk.uuid = cls.message['message']['passback_parameters'][
'disk_uuid']
cls.disk.get_by('uuid')
if state == ResponseState.success.value:
cls.disk.guest_uuid = uuid
cls.disk.sequence = cls.message['message'][
'passback_parameters']['sequence']
cls.disk.state = DiskState.mounted.value
cls.disk.update()
elif action == 'detach_disk':
cls.disk.uuid = cls.message['message']['passback_parameters'][
'disk_uuid']
cls.disk.get_by('uuid')
if state == ResponseState.success.value:
cls.disk.guest_uuid = ''
cls.disk.sequence = -1
cls.disk.state = DiskState.idle.value
cls.disk.update()
elif action == 'boot':
if state == ResponseState.success.value:
pass
elif _object == 'disk':
if action == 'create':
cls.disk.uuid = uuid
cls.disk.get_by('uuid')
cls.disk.node_id = node_id
if state == ResponseState.success.value:
cls.disk.state = DiskState.idle.value
else:
cls.disk.state = DiskState.dirty.value
cls.disk.update()
elif action == 'resize':
if state == ResponseState.success.value:
cls.config.get()
cls.disk.uuid = uuid
cls.disk.get_by('uuid')
cls.disk.size = cls.message['message'][
'passback_parameters']['size']
cls.disk.quota(config=cls.config)
cls.disk.update()
elif action == 'delete':
cls.disk.uuid = uuid
cls.disk.get_by('uuid')
cls.disk.delete()
elif _object == 'snapshot':
if action == 'create':
cls.snapshot.id = cls.message['message'][
'passback_parameters']['id']
cls.snapshot.get()
if state == ResponseState.success.value:
cls.snapshot.snapshot_id = data['snapshot_id']
cls.snapshot.parent_id = data['parent_id']
cls.snapshot.xml = data['xml']
cls.snapshot.progress = 100
cls.snapshot.update()
disks, _ = Disk.get_by_filter(filter_str='guest_uuid:eq:' +
cls.snapshot.guest_uuid)
for disk in disks:
cls.snapshot_disk_mapping.snapshot_id = cls.snapshot.snapshot_id
cls.snapshot_disk_mapping.disk_uuid = disk['uuid']
cls.snapshot_disk_mapping.create()
else:
cls.snapshot.progress = 255
cls.snapshot.update()
if action == 'delete':
if state == ResponseState.success.value:
cls.snapshot.id = cls.message['message'][
'passback_parameters']['id']
cls.snapshot.get()
# 更新子快照的 parent_id 为，当前快照的 parent_id。因为当前快照已被删除。
Snapshot.update_by_filter(
{'parent_id': cls.snapshot.parent_id},
filter_str='parent_id:eq:' + cls.snapshot.snapshot_id)
SnapshotDiskMapping.delete_by_filter(filter_str=':'.join(
['snapshot_id', 'eq', cls.snapshot.snapshot_id]))
cls.snapshot.delete()
else:
pass
if action == 'revert':
# 不论恢复成功与否，都使快照恢复至正常状态。
cls.snapshot.id = cls.message['message'][
'passback_parameters']['id']
cls.snapshot.get()
cls.snapshot.progress = 100
cls.snapshot.update()
if action == 'convert':
cls.snapshot.snapshot_id = cls.message['message'][
'passback_parameters']['id']
cls.snapshot.get_by('snapshot_id')
cls.snapshot.progress = 100
cls.snapshot.update()
cls.os_template_image.id = cls.message['message'][
'passback_parameters']['os_template_image_id']
cls.os_template_image.get()
if state == ResponseState.success.value:
cls.os_template_image.progress = 100
else:
cls.os_template_image.progress = 255
cls.os_template_image.update()
elif _object == 'os_template_image':
if action == 'delete':
cls.os_template_image.id = cls.message['message'][
'passback_parameters']['id']
cls.os_template_image.get()
if state == ResponseState.success.value:
cls.os_template_image.delete()
else:
pass
else:
pass
@classmethod
def guest_collection_performance_processor(cls):
data_kind = cls.message['type']
timestamp = ji.Common.ts()
timestamp -= (timestamp % 60)
data = cls.message['message']['data']
if data_kind == GuestCollectionPerformanceDataKind.cpu_memory.value:
for item in data:
cls.guest_cpu_memory.guest_uuid = item['guest_uuid']
cls.guest_cpu_memory.cpu_load = item['cpu_load']
cls.guest_cpu_memory.memory_available = item[
'memory_available']
cls.guest_cpu_memory.memory_unused = item['memory_unused']
cls.guest_cpu_memory.timestamp = timestamp
cls.guest_cpu_memory.create()
if data_kind == GuestCollectionPerformanceDataKind.traffic.value:
for item in data:
cls.guest_traffic.guest_uuid = item['guest_uuid']
cls.guest_traffic.name = item['name']
cls.guest_traffic.rx_bytes = item['rx_bytes']
cls.guest_traffic.rx_packets = item['rx_packets']
cls.guest_traffic.rx_errs = item['rx_errs']
cls.guest_traffic.rx_drop = item['rx_drop']
cls.guest_traffic.tx_bytes = item['tx_bytes']
cls.guest_traffic.tx_packets = item['tx_packets']
cls.guest_traffic.tx_errs = item['tx_errs']
cls.guest_traffic.tx_drop = item['tx_drop']
cls.guest_traffic.timestamp = timestamp
cls.guest_traffic.create()
if data_kind == GuestCollectionPerformanceDataKind.disk_io.value:
for item in data:
cls.guest_disk_io.disk_uuid = item['disk_uuid']
cls.guest_disk_io.rd_req = item['rd_req']
cls.guest_disk_io.rd_bytes = item['rd_bytes']
cls.guest_disk_io.wr_req = item['wr_req']
cls.guest_disk_io.wr_bytes = item['wr_bytes']
cls.guest_disk_io.timestamp = timestamp
cls.guest_disk_io.create()
else:
pass
@classmethod
def host_collection_performance_processor(cls):
data_kind = cls.message['type']
timestamp = ji.Common.ts()
timestamp -= (timestamp % 60)
data = cls.message['message']['data']
if data_kind == HostCollectionPerformanceDataKind.cpu_memory.value:
cls.host_cpu_memory.node_id = data['node_id']
cls.host_cpu_memory.cpu_load = data['cpu_load']
cls.host_cpu_memory.memory_available = data['memory_available']
cls.host_cpu_memory.timestamp = timestamp
cls.host_cpu_memory.create()
if data_kind == HostCollectionPerformanceDataKind.traffic.value:
for item in data:
cls.host_traffic.node_id = item['node_id']
cls.host_traffic.name = item['name']
cls.host_traffic.rx_bytes = item['rx_bytes']
cls.host_traffic.rx_packets = item['rx_packets']
cls.host_traffic.rx_errs = item['rx_errs']
cls.host_traffic.rx_drop = item['rx_drop']
cls.host_traffic.tx_bytes = item['tx_bytes']
cls.host_traffic.tx_packets = item['tx_packets']
cls.host_traffic.tx_errs = item['tx_errs']
cls.host_traffic.tx_drop = item['tx_drop']
cls.host_traffic.timestamp = timestamp
cls.host_traffic.create()
if data_kind == HostCollectionPerformanceDataKind.disk_usage_io.value:
for item in data:
cls.host_disk_usage_io.node_id = item['node_id']
cls.host_disk_usage_io.mountpoint = item['mountpoint']
cls.host_disk_usage_io.used = item['used']
cls.host_disk_usage_io.rd_req = item['rd_req']
cls.host_disk_usage_io.rd_bytes = item['rd_bytes']
cls.host_disk_usage_io.wr_req = item['wr_req']
cls.host_disk_usage_io.wr_bytes = item['wr_bytes']
cls.host_disk_usage_io.timestamp = timestamp
cls.host_disk_usage_io.create()
else:
pass
@classmethod
def launch(cls):
logger.info(msg='Thread EventProcessor is launched.')
while True:
if Utils.exit_flag:
msg = 'Thread EventProcessor say bye-bye'
print msg
logger.info(msg=msg)
return
try:
report = db.r.lpop(app.config['upstream_queue'])
if report is None:
time.sleep(1)
continue
cls.message = json.loads(report)
if cls.message['kind'] == EmitKind.log.value:
cls.log_processor()
elif cls.message['kind'] == EmitKind.guest_event.value:
cls.guest_event_processor()
elif cls.message['kind'] == EmitKind.host_event.value:
cls.host_event_processor()
elif cls.message['kind'] == EmitKind.response.value:
cls.response_processor()
elif cls.message[
'kind'] == EmitKind.guest_collection_performance.value:
cls.guest_collection_performance_processor()
elif cls.message[
'kind'] == EmitKind.host_collection_performance.value:
cls.host_collection_performance_processor()
else:
pass
except Exception as e:
logger.error(traceback.format_exc())
time.sleep(1)
