def run(self):
meta = json.loads(self.task.task_def.json_meta)
url = None
if (self.task.task_def.task_type == 'snapshot'):
name = ('%s_%d' % (meta['prefix'], int(time.time())))
url = ('%sshares/%s/snapshots/%s' % (self.baseurl, meta['share'],
name))
elif (self.task.task_def.task_type == 'scrub'):
url = ('%spools/%s/scrub' % (self.baseurl, meta['pool']))
try:
api_call(url, data=None, calltype='post')
if (self.task.task_def.task_type == 'scrub'):
url = ('%spools/%s/scrub/status' % (self.baseurl,
meta['pool']))
while (True):
status = api_call(url, data=None, calltype='post')
logger.info('scrub status: %s' % status)
if (status['status'] == 'finished'):
break
time.sleep(10)
except Exception, e:
logger.exception(e)
sys.exit(3)
def main():
tid = int(sys.argv[1])
tdo = TaskDefinition.objects.get(id=tid)
if (tdo.task_type != 'scrub'):
return logger.error('task_type(%s) is not scrub.' % tdo.task_type)
meta = json.loads(tdo.json_meta)
if (Task.objects.filter(task_def=tdo).exists()):
ll = Task.objects.filter(task_def=tdo).order_by('-id')[0]
if (ll.state != 'error' and ll.state != 'finished'):
logger.debug('Non terminal state(%s) for task(%d). Checking '
'again.' % (ll.state, tid))
cur_state = update_state(ll, meta['pool'])
if (cur_state != 'error' and cur_state != 'finished'):
return logger.debug('Non terminal state(%s) for task(%d). '
'A new task will not be run.' % (cur_state, tid))
now = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=utc)
t = Task(task_def=tdo, state='started', start=now)
url = ('%spools/%s/scrub' % (baseurl, meta['pool']))
try:
api_call(url, data=None, calltype='post', save_error=False)
logger.debug('Started scrub at %s' % url)
t.state = 'running'
except Exception, e:
logger.error('Failed to start scrub at %s' % url)
t.state = 'error'
logger.exception(e)
def main():
tid = int(sys.argv[1])
tdo = TaskDefinition.objects.get(id=tid)
if (tdo.task_type != 'scrub'):
return logger.error('task_type(%s) is not scrub.' % tdo.task_type)
meta = json.loads(tdo.json_meta)
if (Task.objects.filter(task_def=tdo).exists()):
ll = Task.objects.filter(task_def=tdo).order_by('-id')[0]
if (ll.state != 'error' and ll.state != 'finished'):
logger.debug('Non terminal state(%s) for task(%d). Checking '
'again.' % (ll.state, tid))
cur_state = update_state(ll, meta['pool'])
if (cur_state != 'error' and cur_state != 'finished'):
return logger.debug('Non terminal state(%s) for task(%d). '
'A new task will not be run.' %
(cur_state, tid))
now = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=utc)
t = Task(task_def=tdo, state='started', start=now)
url = ('%spools/%s/scrub' % (baseurl, meta['pool']))
try:
api_call(url, data=None, calltype='post', save_error=False)
logger.debug('Started scrub at %s' % url)
t.state = 'running'
except Exception, e:
logger.error('Failed to start scrub at %s' % url)
t.state = 'error'
logger.exception(e)
def run(self):
meta = json.loads(self.task.task_def.json_meta)
url = None
if (self.task.task_def.task_type == 'snapshot'):
name = ('%s_%d' % (meta['prefix'], int(time.time())))
url = ('%sshares/%s/snapshots/%s' %
(self.baseurl, meta['share'], name))
elif (self.task.task_def.task_type == 'scrub'):
url = ('%spools/%s/scrub' % (self.baseurl, meta['pool']))
try:
api_call(url, data=None, calltype='post')
if (self.task.task_def.task_type == 'scrub'):
url = ('%spools/%s/scrub/status' %
(self.baseurl, meta['pool']))
while (True):
status = api_call(url, data=None, calltype='post')
logger.info('scrub status: %s' % status)
if (status['status'] == 'finished'):
break
time.sleep(10)
except Exception, e:
logger.exception(e)
sys.exit(3)
def update_storage_state(self):
resources = [
{
'url': '%s/disks/scan' % self.base_url,
'success': 'Disk state updated successfully',
'error': 'Failed to update disk state.'
},
{
'url': '%s/commands/refresh-pool-state' % self.base_url,
'success': 'Pool state updated successfully',
'error': 'Failed to update pool state.'
},
{
'url': '%s/commands/refresh-share-state' % self.base_url,
'success': 'Share state updated successfully',
'error': 'Failed to update share state.'
},
{
'url': '%s/commands/refresh-snapshot-state' % self.base_url,
'success': 'Snapshot state updated successfully',
'error': 'Failed to update snapshot state.'
},
]
for r in resources:
try:
api_call(r['url'],
data=None,
calltype='post',
save_error=False)
logger.debug(r['success'])
except Exception, e:
logger.error('%s. exception: %s' % (r['error'], e.__str__()))
def main():
tid = int(sys.argv[1])
tdo = TaskDefinition.objects.get(id=tid)
stype = 'task_scheduler'
baseurl = 'https://localhost/api/'
if (tdo.task_type != 'snapshot'):
logger.error('task_type(%s) is not snapshot.' % tdo.task_type)
return
meta = json.loads(tdo.json_meta)
validate_snap_meta(meta)
max_count = int(float(meta['max_count']))
share = Share.objects.get(name=meta['share'])
prefix = ('%s_' % meta['prefix'])
snapshots = Snapshot.objects.filter(share=share, snap_type=stype,
name__startswith=prefix).order_by('-id')
if (len(snapshots) > max_count):
for snap in snapshots[max_count:]:
url = ('%s/shares/%s/snapshots/%s' %
(baseurl, meta['share'], snap.name))
try:
api_call(url, data=None, calltype='delete', save_error=False)
logger.debug('deleted old snapshot at %s' % url)
except Exception, e:
logger.error('Failed to delete old snapshot at %s' % url)
logger.exception(e)
return
def main():
tid = int(sys.argv[1])
tdo = TaskDefinition.objects.get(id=tid)
stype = 'task_scheduler'
baseurl = 'https://localhost/api/'
if (tdo.task_type != 'snapshot'):
logger.error('task_type(%s) is not snapshot.' % tdo.task_type)
return
meta = json.loads(tdo.json_meta)
validate_snap_meta(meta)
max_count = int(float(meta['max_count']))
share = Share.objects.get(name=meta['share'])
prefix = ('%s_' % meta['prefix'])
snapshots = Snapshot.objects.filter(
share=share, snap_type=stype, name__startswith=prefix).order_by('-id')
if (len(snapshots) > max_count):
for snap in snapshots[max_count:]:
url = ('%s/shares/%s/snapshots/%s' %
(baseurl, meta['share'], snap.name))
try:
api_call(url, data=None, calltype='delete', save_error=False)
logger.debug('deleted old snapshot at %s' % url)
except Exception, e:
logger.error('Failed to delete old snapshot at %s' % url)
logger.exception(e)
return
def update_rockons(self):
try:
url = '%s/rockons/update' % self.base_url
api_call(url, data=None, calltype='post', save_error=False)
logger.debug('Updated Rock-on metadata')
except Exception, e:
logger.debug('failed to update Rock-on metadata. low-level '
'exception: %s' % e.__str__())
def disable_replica(rid, logger):
try:
url = "%s/sm/replicas/%d" % (BASE_URL, rid)
api_call(url, data={"enabled": False}, calltype="put", save_error=False)
return logger.info("Replica(%s) is disabled" % url)
except Exception, e:
logger.error("Failed to disable replica(%s)" % url)
raise e
def update_rockons(self):
try:
url = '%s/rockons/update' % self.base_url
api_call(url, data=None, calltype='post', save_error=False)
logger.debug('Updated Rock-on metadata')
except Exception, e:
logger.debug('failed to update Rock-on metadata. low-level '
'exception: %s' % e.__str__())
def update_replica_status(rid, data, logger):
try:
url = "%ssm/replicas/trail/%d" % (BASE_URL, rid)
api_call(url, data=data, calltype="put", save_error=False)
return logger.info("replica(%s) status updated to %s" % (url, data["status"]))
except Exception, e:
logger.error("Failed to update replica(%s) status to: %s" % (url, data["status"]))
raise e
def is_share(sname, logger):
try:
url = ('%sshares/%s' % (BASE_URL, sname))
api_call(url, save_error=False)
return True
except Exception, e:
logger.error('Exception while looking up if share exists at: %s '
'. Exception: %s' % (url, e.__str__()))
return False
def update_rockons(self):
from cli.rest_util import api_call
try:
url = 'https://localhost/api/rockons/update'
api_call(url, data=None, calltype='post', save_error=False)
logger.debug('Updated Rock-on metadata')
except Exception, e:
logger.debug('failed to update Rock-on metadata. low-level '
'exception: %s' % e.__str__())
def main():
# bootstrap the machine. success of quit
url = 'https://localhost/api/commands/bootstrap'
time.sleep(10)
try:
api_call(url, calltype='post')
except Exception, e:
logger.error('Unable to bootstrap the machine. Moving on..')
logger.exception(e)
def is_share(sname, logger):
try:
url = ('%sshares/%s' % (BASE_URL, sname))
api_call(url, save_error=False)
return True
except Exception, e:
logger.error('Exception while looking up if share exists at: %s' % url)
logger.exception(e)
return False
def disable_replica(rid, logger):
try:
url = ('%s/sm/replicas/%d' % (BASE_URL, rid))
api_call(url, data={'enabled': False, }, calltype='put',
save_error=False)
return logger.info('Replica(%s) is disabled' % url)
except Exception, e:
logger.error('Failed to disable replica(%s)' % url)
raise e
def update_trail(tid, data, logger):
try:
url = ('%splugin/backup/trail/%d' % (BASE_URL, tid))
api_call(url, data=data, calltype='put', save_error=False)
return logger.info('policy trail(%s) status updated to %s' %
(url, data['status']))
except Exception, e:
logger.error('Failed to update policy trail(%s) status to %s' %
(url, data['status']))
raise e
def update_trail(tid, data, logger):
try:
url = ('%splugin/backup/trail/%d' % (BASE_URL, tid))
api_call(url, data=data, calltype='put', save_error=False)
return logger.info('policy trail(%s) status updated to %s' %
(url, data['status']))
except Exception, e:
logger.error('Failed to update policy trail(%s) status to %s' %
(url, data['status']))
raise e
def update_replica_status(rid, data, logger):
try:
url = ('%ssm/replicas/trail/%d' % (BASE_URL, rid))
api_call(url, data=data, calltype='put', save_error=False)
return logger.info('replica(%s) status updated to %s' %
(url, data['status']))
except Exception, e:
logger.error('Failed to update replica(%s) status to: %s'
% (url, data['status']))
raise e
def is_snapshot(sname, snap_name, logger):
try:
# do a get and see if the snapshot is already created
url = ('%sshares/%s/snapshots/%s' % (BASE_URL, sname, snap_name))
api_call(url, save_error=False)
return True
except RockStorAPIException, e:
if (re.match('Invalid api end point', e.detail) is not None):
# it's 404.
return False
raise e
def is_snapshot(sname, snap_name, logger):
try:
# do a get and see if the snapshot is already created
url = ('%sshares/%s/snapshots/%s' % (BASE_URL, sname, snap_name))
api_call(url, save_error=False)
return True
except RockStorAPIException, e:
if (re.match('Invalid api end point', e.detail) is not None):
# it's 404.
return False
raise e
def generic_post(url, payload):
headers = {'content-type': 'application/json', }
try:
api_call(url, data=payload, calltype='post', headers=headers,
save_error=False)
logger.debug('Successfully created resource: {}. '
'Payload: {}'.format(url, payload))
except Exception as e:
logger.error('Exception occurred while creating resource: {}. '
'Payload: {}. Exception: {}. '
'Moving on.'.format(url, payload, e.__str__()))
def main():
# bootstrap the machine. success of quit
bootstrap_url = 'https://localhost/api/commands/bootstrap'
diskscan_url = 'https://localhsot/api/disks/scan'
time.sleep(10)
try:
set_token()
api_call(diskscan_url, calltype='post')
api_call(bootstrap_url, calltype='post')
except Exception, e:
logger.error('Unable to bootstrap the machine. Moving on..')
logger.exception(e)
def delete_old_snapshots(sname, num_retain, logger):
url = ('%sshares/%s/snapshots' % (BASE_URL, sname))
snap_details = api_call(url, save_error=False)
if (snap_details['count'] > num_retain):
extra = snap_details['count'] - num_retain
for i in range(extra):
if (i >= len(snap_details['results'])):
return delete_old_snapshots(sname, num_retain, logger)
snap_name = snap_details['results'][i]['name']
api_call('%s/%s' % (url, snap_name), calltype='delete',
save_error=False)
logger.debug('deleted snap: %s' % snap_name)
def generic_post(url, payload):
headers = {"content-type": "application/json"}
try:
api_call(url, data=payload, calltype="post", headers=headers, save_error=False)
logger.debug(
"Successfully created resource: {}. Payload: {}".format(url, payload)
)
except Exception as e:
logger.error(
"Exception occurred while creating resource: {}. "
"Payload: {}. Exception: {}. "
"Moving on.".format(url, payload, e.__str__())
)
def delete_old_snapshots(sname, num_retain, logger):
url = ('%sshares/%s/snapshots' % (BASE_URL, sname))
snap_details = api_call(url, save_error=False)
if (snap_details['count'] > num_retain):
extra = snap_details['count'] - num_retain
for i in range(extra):
if (i >= len(snap_details['results'])):
return delete_old_snapshots(sname, num_retain, logger)
snap_name = snap_details['results'][i]['name']
api_call('%s/%s' % (url, snap_name),
calltype='delete',
save_error=False)
logger.debug('deleted snap: %s' % snap_name)
def main():
context = zmq.Context()
pull_socket = context.socket(zmq.PULL)
pull_socket.RCVTIMEO = 500
pull_socket.bind('tcp://%s:%d' % settings.SPROBE_SINK)
#bootstrap the machine. success of quit
url = 'https://localhost/api/commands/bootstrap'
time.sleep(5)
try:
api_call(url, calltype='post')
except Exception, e:
logger.error('Unable to bootstrap the machine. Aborting')
logger.exception(e)
return -1
def _replication_interface(self):
url = 'https://localhost/api/network'
interfaces = api_call(url, save_error=False)
mgmt_iface = [
x for x in interfaces['results'] if x['itype'] == 'management'
][0]
return mgmt_iface['ipaddr']
def create_snapshot(sname, snap_name, logger, snap_type="replication"):
try:
url = "%sshares/%s/snapshots/%s" % (BASE_URL, sname, snap_name)
snap_details = api_call(url, data={"snap_type": snap_type}, calltype="post", save_error=False)
return logger.debug("created snapshot. url: %s. details = %s" % (url, snap_details))
except RockStorAPIException, e:
return logger.debug("Failed to create snapshot: %s. It may already " "exist. error: %s" % (url, e.detail))
def generic_post(url, payload):
headers = {
'content-type': 'application/json',
}
try:
api_call(url,
data=payload,
calltype='post',
headers=headers,
save_error=False)
logger.debug('Successfully created resource: %s. payload: %s' %
(url, payload))
except Exception, e:
logger.error('Exception occured while creating resource: %s. '
'payload: %s. exception: %s. Moving on.' %
(url, payload, e.__str__()))
def rshare_id(sname, logger):
try:
url = ('%ssm/replicas/rshare/%s' % (BASE_URL, sname))
rshare = api_call(url, save_error=False)
return rshare['id']
except Exception:
raise
def create_receive_trail(rid, data, logger):
url = ('%ssm/replicas/rtrail/rshare/%d' % (BASE_URL, rid))
try:
rt = api_call(url, data=data, calltype='post', save_error=False)
return rt['id']
except Exception:
raise
def prune_trail(url, logger, days=7):
try:
data = {'days': days, }
return api_call(url, data=data, calltype='delete', save_error=False)
except Exception, e:
logger.debug('Failed to prune trail for url(%s)' % url)
logger.exception(e)
def update_receive_trail(rtid, data, logger):
url = ('%ssm/replicas/rtrail/%d' % (BASE_URL, rtid))
try:
return api_call(url, data=data, calltype='put', save_error=False)
except Exception:
logger.error('Failed to update receive trail: %s' % url)
raise
def update_receive_trail(rtid, data, logger):
url = ('%ssm/replicas/rtrail/%d' % (BASE_URL, rtid))
try:
return api_call(url, data=data, calltype='put', save_error=False)
except Exception:
logger.error('Failed to update receive trail: %s' % url)
raise
def create_receive_trail(rid, data, logger):
url = ('%ssm/replicas/rtrail/rshare/%d' % (BASE_URL, rid))
try:
rt = api_call(url, data=data, calltype='post', save_error=False)
return rt['id']
except Exception:
raise
def run(self):
meta = json.loads(self.task.json_meta)
url = None
if (self.task.name == 'snapshot'):
name = ('%s_%d' % (meta['prefix'], int(time.time())))
url = ('%sshares/%s/snapshots/%s' %
(self.baseurl, meta['share'], name))
elif (self.task.name == 'scrub'):
url = ('%spools/%s/scrub/' % (self.baseurl, meta['pool']))
try:
api_call(url, data=None, calltype='post')
except Exception, e:
logger.exception(e)
sys.exit(3)
def run(self):
meta = json.loads(self.task.json_meta)
url = None
if (self.task.name == 'snapshot'):
name = ('%s_%d' % (meta['prefix'], int(time.time())))
url = ('%sshares/%s/snapshots/%s' % (self.baseurl, meta['share'],
name))
elif (self.task.name == 'scrub'):
url = ('%spools/%s/scrub/' % (self.baseurl, meta['pool']))
try:
api_call(url, data=None, calltype='post')
except Exception, e:
logger.exception(e)
sys.exit(3)
def rshare_id(sname, logger):
try:
url = ('%ssm/replicas/rshare/%s' % (BASE_URL, sname))
rshare = api_call(url, save_error=False)
return rshare['id']
except Exception:
raise
def _replication_interface(self):
url = 'https://localhost/api/network'
interfaces = api_call(url, save_error=False)
logger.info('interfaces: %s' % interfaces)
mgmt_iface = [x for x in interfaces['results']
if x['itype'] == 'management'][0]
logger.info('mgmt_iface: %s' % mgmt_iface)
return mgmt_iface['ipaddr']
def delete_snapshot(sname, snap_name, logger):
try:
url = ('%sshares/%s/snapshots/%s' % (BASE_URL, sname, snap_name))
return api_call(url, calltype='delete', save_error=False)
except RockStorAPIException, e:
if (e.detail == 'Snapshot(%s) does not exist.' % snap_name):
return logger.debug(e.detail)
raise e
def update_replica_status(rtid, data, logger):
try:
url = ('%ssm/replicas/trail/%d' % (BASE_URL, rtid))
return api_call(url, data=data, calltype='put')
except Exception, e:
logger.error('Failed to update replica(%s) status to: %s' %
(url, data['status']))
raise e
def rshare_id(sname, logger):
try:
url = "%ssm/replicas/rshare/%s" % (BASE_URL, sname)
rshare = api_call(url, save_error=False)
logger.debug("rshare exists: %s" % rshare)
return rshare["id"]
except Exception:
raise
def update_receive_trail(rtid, data, logger):
url = "%s/sm/replicas/rtrail/%d" % (BASE_URL, rtid)
try:
rt = api_call(url, data=data, calltype="put", save_error=False)
return logger.debug("Updated receive trail: %s" % rt)
except Exception:
logger.error("Failed to update receive trail: %s" % url)
raise
def delete_snapshot(sname, snap_name, logger):
try:
url = ('%sshares/%s/snapshots/%s' % (BASE_URL, sname, snap_name))
return api_call(url, calltype='delete', save_error=False)
except RockStorAPIException, e:
if (e.detail == 'Snapshot(%s) does not exist.' % snap_name):
return logger.debug(e.detail)
raise e
def update_replica_status(rtid, data, logger):
try:
url = ('%ssm/replicas/trail/%d' % (BASE_URL, rtid))
return api_call(url, data=data, calltype='put')
except Exception, e:
logger.error('Failed to update replica(%s) status to: %s'
% (url, data['status']))
raise e
def create_snapshot(sname, snap_name, logger):
try:
url = ('%sshares/%s/snapshots/%s' % (BASE_URL, sname, snap_name))
snap_details = api_call(url, calltype='post', save_error=False)
return logger.debug('created snapshot. url: %s. details = %s' %
(url, snap_details))
except RockStorAPIException, e:
return logger.debug('Failed to create snapshot: %s. It may already '
'exist. error: %s' % (url, e.detail))
def update_state(t, pool):
url = ('%spools/%s/scrub/status' % (baseurl, pool))
try:
status = api_call(url, data=None, calltype='post', save_error=False)
t.state = status['status']
except Exception, e:
logger.error('Failed to get scrub status at %s' % url)
t.state = 'error'
logger.exception(e)
def create_trail(pid, logger):
try:
url = ('%splugin/backup/trail/policy/%d' % (BASE_URL, pid))
td = api_call(url, calltype='post', save_error=False)
logger.info('new policy trail created for policy(%d)' % pid)
return td
except Exception, e:
logger.error('Failed to create new policy trail for policy(%d)' % pid)
raise e
def create_replica_trail(rid, snap_name, logger):
url = ('%ssm/replicas/trail/replica/%d' % (BASE_URL, rid))
try:
rt = api_call(url, data={'snap_name': snap_name, },
calltype='post', save_error=False)
return rt
except Exception, e:
logger.error('Failed to create replica trail: %s' % url)
raise e
def disable_replica(rid, logger):
try:
url = ('%ssm/replicas/%d' % (BASE_URL, rid))
headers = {'content-type': 'application/json', }
return api_call(url, data={'enabled': False, }, calltype='put',
save_error=False, headers=headers)
except Exception, e:
logger.error('Failed to disable replica(%s)' % url)
raise e
def prune_trail(url, logger, days=7):
try:
data = {
'days': days,
}
return api_call(url, data=data, calltype='delete', save_error=False)
except Exception, e:
logger.debug('Failed to prune trail for url(%s)' % url)
logger.exception(e)
def create_rshare(data, logger):
try:
url = ('%ssm/replicas/rshare' % BASE_URL)
rshare = api_call(url, data=data, calltype='post', save_error=False)
logger.debug('ReplicaShare: %s created.' % rshare)
return rshare['id']
except RockStorAPIException, e:
return logger.debug('Failed to create rshare: %s. It may already'
' exists. error: %s' % (url, e.detail))
def create_receive_trail(rid, data, logger):
url = ('%s/sm/replicas/rtrail/rshare/%d' % (BASE_URL, rid))
try:
rt = api_call(url, data=data, calltype='post', save_error=False)
logger.debug('Created receive trail: %s' % rt)
return rt['id']
except Exception:
logger.error('Failed to create receive trail: %s' % url)
raise
def get_sender_ip(uuid, logger):
try:
url = ('%sappliances' % BASE_URL)
ad = api_call(url, save_error=False)
for a in ad['results']:
if (a['uuid'] == uuid):
return a['ip']
except Exception, e:
logger.error('Failed to get sender ip address')
raise e
def create_replica_trail(rid, snap_name, logger):
url = ('%ssm/replicas/trail/replica/%d' % (BASE_URL, rid))
try:
rt = api_call(url, data={'snap_name': snap_name, },
calltype='post', save_error=False)
logger.debug('Created replica trail: %s' % url)
return rt
except Exception, e:
logger.error('Failed to create replica trail: %s' % url)
raise e
def generic_post(url, payload):
headers = {"content-type": "application/json"}
try:
api_call(url,
data=payload,
calltype="post",
headers=headers,
save_error=False)
# Remove "password" indexed items from our dict payload to avoid logging them.
if isinstance(payload, dict):
payload.pop("password", None)
else:
logger.info("Non Dictionary payload")
logger.info("Successfully created resource: {}. Payload: {}".format(
url, payload))
except Exception as e:
logger.error("Exception occurred while creating resource: {}. "
"Payload: {}. Exception: {}. "
"Moving on.".format(url, payload, e.__str__()))
