def system_info(request):
def human_bytes(bytes):
#source: https://github.com/bartTC/django-memcache-status
bytes = float(bytes)
if bytes >= 1073741824:
gigabytes = bytes / 1073741824
size = '%.2fGB' % gigabytes
elif bytes >= 1048576:
megabytes = bytes / 1048576
size = '%.2fMB' % megabytes
elif bytes >= 1024:
kilobytes = bytes / 1024
size = '%.2fKB' % kilobytes
else:
size = '%.2fB' % bytes
return size
context = get_hqadmin_base_context(request)
context['couch_update'] = request.GET.get('couch_update', 5000)
context['celery_update'] = request.GET.get('celery_update', 10000)
context['hide_filters'] = True
if hasattr(os, 'uname'):
context['current_system'] = os.uname()[1]
#from dimagi.utils import gitinfo
#context['current_ref'] = gitinfo.get_project_info()
#removing until the async library is updated
context['current_ref'] = {}
if settings.COUCH_USERNAME == '' and settings.COUCH_PASSWORD == '':
couchlog_resource = Resource("http://%s/" % (settings.COUCH_SERVER_ROOT))
else:
couchlog_resource = Resource("http://%s:%s@%s/" % (settings.COUCH_USERNAME, settings.COUCH_PASSWORD, settings.COUCH_SERVER_ROOT))
try:
#todo, fix on bigcouch/cloudant
context['couch_log'] = "Will be back online shortly" if is_bigcouch() \
else couchlog_resource.get('_log', params_dict={'bytes': 2000 }).body_string()
except Exception, ex:
context['couch_log'] = "unable to open couch log: %s" % ex
def _delete_page(self, path):
"""delete page at a given path"""
retval = None
res = Resource(self.host)
page = res.delete(path)
data = page.body_string()
if data:
retval = json.loads(data)
return retval
def __init__(self, bot):
IPlugin.__init__(self, bot)
self.data = None
self.config = None
self.url = None
self.ignored = []
self.auth = None
self.project_id = None
self.resource = None
self.lastCheck = 0
self.config = globalConfig.get('plugins.redmine')
if self.config is None:
logging.error('Redmine: Disabled.')
return
self.data = {
'last-bug-created': 0,
'ignored-names': [
'/^Not\-[0-9]+/' # Notifico bots
]
}
self.LoadPluginData()
self.url = globalConfig.get('plugins.redmine.url', None)
if self.url is None:
logging.error('Redmine: Disabled.')
return
self.ignored = []
for ignoretok in self.data.get('ignored-names', ['/^Not\-[0-9]/']):
if ignoretok.startwith('/') and ignoretok.endwith('/'):
self.ignored += [re.compile(ignoretok[1:-1])]
else:
self.ignored += [re.compile('^' + re.escape(ignoretok) + '$')]
self.auth = BasicAuth(globalConfig.get('plugins.redmine.apikey', None),
str(random.random()))
self.project_id = globalConfig.get('plugins.redmine.project', None)
if self.project_id is None:
logging.warning('Redmine: Not going to check for bug updates.')
self.bug_info_format = globalConfig.get(
'plugins.redmine.bug-info-format',
'Redmine #{ID} - {AUTHOR} - {STATUS} - {SUBJECT}{CRLF}{URL}')
self.new_bug_format = globalConfig.get(
'plugins.redmine.new-bug-format',
'NEW ISSUE: {URL} (#{ID}: {SUBJECT})')
self.resource = Resource(self.url, filters=[self.auth])
self.bug_regex = re.compile(r'#(\d+)\b')
self.lastCheck = 0
def getJSON(self, url):
resource = Resource(url, filters=[auth])
response = resource.get(headers={'Content-Type': 'application/json'})
if response.status_int == 200:
# Not all resources will return 200 on success. There are other success status codes. Like 204. We've read
# the documentation for though and know what to expect here.
versions = json.loads(response.body_string())
return versions
else:
print response.status_int
# print response.
return None
def unauthorizedChecks(self, users, path, method="GET", params_dict=None):
""":param users: list of BasicAuths to try, expecting Unauthorized"""
for user in users:
name = "None"
if (user):
name = user.credentials[0]
exception = False
try:
res = Resource(self.baseurl, filters=[user])
output = res.request(method, path, params_dict=params_dict)
except Unauthorized:
exception = True
self.assertTrue(exception, "No unauthorized exception for " + name)
def testGeneralUpdate(self):
path = "/resources/conf/general"
for user in [self.testconfiguser, self.testpoweruser, self.testsuperuser]:
newinterval = random.randrange(300, 600)
general = self.getJSONResource(path, user)
general['interval'] = newinterval
res = Resource(self.baseurl, filters=[user])
output = res.request("PUT", path=path, params_dict=general)
self.assertEqual(204, output.status_int, "Wrong response code: " + user.credentials[0])
general = self.getJSONResource(path, user)
self.assertEquals(newinterval, int(general['interval']), "General not updated.")
self.unauthorizedChecks([None, self.testuser, self.testfilluser], path, method="PUT")
def is_pullrequest_tested(pr,token,owner,repo):
url='https://api.github.com/repos/'+owner+'/'+repo+'/pulls/'+pr+'/files'
resource = Resource(url, pool=pool)
headers = {'Content-Type' : 'application/json' }
headers['Authorization'] = 'token %s' % token
response = resource.get(headers = headers)
files = json.loads(response.body_string())
#print 'NORMAL:', json.dumps(files, sort_keys=True,indent=2)
for x in files:
if 'test' in x['filename'].lower():
return True;
return False;
def __init__(self, irc):
self.__parent = super(Redmine, self)
self.__parent.__init__(irc)
self.saidBugs = ircutils.IrcDict()
sayTimeout = self.registryValue('bugSnarferTimeout')
for k in irc.state.channels.keys():
self.saidBugs[k] = TimeoutQueue(sayTimeout)
self.url = self.registryValue('urlbase')
self.auth = BasicAuth(self.registryValue('apikey'),
str(random.random()))
self.resource = Resource(self.url, filters=[self.auth])
def get_issue(key):
""" Given an issue key (i.e. JRA-9) return the JSON representation of it. This is the only place where we deal
with JIRA's REST API. """
print('Fetching ' + key)
# we need to expand subtasks and links since that's what we care about here.
resource = Resource(url + ('/rest/api/latest/issue/%s' % key), pool_instance=pool, filters=[auth])
response = resource.get(headers = {'Content-Type' : 'application/json'})
if response.status_int == 200:
# Not all resources will return 200 on success. There are other success status codes. Like 204. We've read
# the documentation for though and know what to expect here.
issue = json.loads(response.body_string())
return issue
else:
return None
def testUpdateByUserForSelf(self):
"""succeed in updating alarm belonging to self"""
path = "/resources/alarms/115" # id taken from DB
user = self.testuser
res = Resource(self.baseurl, filters=[user])
# perform request
output = res.request("PUT",path=path,
params_dict = {"alarm_type_id": 14, # changeing from 2 to 1
"account_id": 104, # std user from the DB
"greater_than_p": True,
"value": 105})
# check that update succeeded
self.assertEqual(202, output.status_int, "User was unable to update alarm for self.")
def testDeleteNonExistantAlarm(self):
"""Power user to fail in deleting non-existant alarm"""
path = "/resources/alarms/8"
user = self.testpoweruser
res = Resource(self.baseurl, filters=[user])
try:
exception = False
output = res.request("DELETE",path=path)
except: # which exception should I catch?
exception = True
self.assertTrue(exception,"Failed to return 404 for missing alarm.")
def iterator(self):
"""
An iterator over the results from applying this QuerySet to the
remote web service.
"""
resource = Resource(self.model.get_resource_url_list(),
filters=ROA_FILTERS,
**ROA_SSL_ARGS)
try:
parameters = self.query.parameters
logger.debug(
u"""Requesting: "%s" through %s with parameters "%s" """ %
(self.model.__name__, resource.uri, force_unicode(parameters)))
response = resource.get(headers=self._get_http_headers(),
**parameters)
except ResourceNotFound:
return
except Exception as e:
raise ROAException(e)
response = force_unicode(
response.body_string()).encode(DEFAULT_CHARSET)
# Deserializing objects:
data = self.model.get_parser().parse(StringIO(response))
# Check limit_start and limit_stop arguments for pagination and only
# slice data if they are both numeric and there are results left to go.
# We only perform this check on lists.
limit_start = getattr(self.query, 'limit_start', None)
limit_stop = getattr(self.query, 'limit_stop', None)
if (isinstance(limit_start, int) and isinstance(limit_stop, int)
and limit_stop - limit_start < len(data)
and limit_stop <= len(data) and isinstance(data, list)):
data = data[limit_start:limit_stop]
# [] is the case of empty no-paginated result
if data != []:
serializer = self.model.get_serializer(data=data)
if not serializer.is_valid():
raise ROAException(
u'Invalid deserialization for %s model: %s' %
(self.model, serializer.errors))
i = 0
for obj in serializer.validated_data:
obj['id'] = serializer.initial_data[i].get('id', None)
i += 1
yield self.model(**obj)
def _delete_page(host, path):
"""delete page at a given path"""
retval = None
if "http://" not in host:
host = "http://{0}".format(host)
res = Resource(host)
page = res.delete(path)
data = page.body_string()
if data:
retval = json.loads(data)
return retval
def testGet(self):
"""Test ability to get list of alarm events"""
path = "/resources/alarms/events"
for user in [self.testuser, self.testpoweruser, self.testsuperuser]:
# perform request
res = Resource(self.baseurl, filters=[user])
output = res.request("GET",path=path,
params_dict = {"begin": "010111134553",
"end": "120111134553"})
self.assertEqual(200,output.status_int, "User, " + str(user) + " was unable to obtain list of alarm events.")
self.unauthorizedChecks([None], path)
def testDeleteByUserForOthers(self):
"""fail to delete other's alarm contact"""
path = "/resources/alarms/alarm_contacts/100" # alarm belonging to a poweruser
user = self.testuser
res = Resource(self.baseurl, filters=[user])
try:
exception = False
output = res.request("DELETE",path=path)
except: # which exception should I catch?
exception = True
self.assertTrue(exception,"Failed to return 404 for missing alarm.")
def testNewByUserForSelf(self):
"""create alarm contact for self"""
path = "/resources/alarms"
user = self.testuser
res = Resource(self.baseurl, filters=[user])
# perform request
output = res.request("POST",path=path,
params_dict = {"alarm_type_id": 10,
"account_id":104,
"alarm_contact_type_ids": 100})
# check if addition failed
self.assertEqual(200, output.status_int, "User was unable to add alarm contact for self.")
# get the returned url
xlink = json.loads(output.body_string())
def system_ajax(request):
"""
Utility ajax functions for polling couch and celerymon
"""
type = request.GET.get('api', None)
task_limit = getattr(settings, 'CELERYMON_TASK_LIMIT', 5)
celerymon_url = getattr(settings, 'CELERYMON_URL', '')
db = XFormInstance.get_db()
ret = {}
if type == "_active_tasks":
tasks = [] if is_bigcouch() else filter(lambda x: x['type'] == "indexer", db.server.active_tasks())
#for reference structure is:
# tasks = [{'type': 'indexer', 'pid': 'foo', 'database': 'mock',
# 'design_document': 'mockymock', 'progress': 0,
# 'started_on': 1349906040.723517, 'updated_on': 1349905800.679458,
# 'total_changes': 1023},
# {'type': 'indexer', 'pid': 'foo', 'database': 'mock',
# 'design_document': 'mockymock', 'progress': 70,
# 'started_on': 1349906040.723517, 'updated_on': 1349905800.679458,
# 'total_changes': 1023}]
return HttpResponse(json.dumps(tasks), mimetype='application/json')
elif type == "_stats":
return HttpResponse(json.dumps({}), mimetype = 'application/json')
elif type == "_logs":
pass
if celerymon_url != '':
cresource = Resource(celerymon_url, timeout=3)
if type == "celerymon_poll":
#inefficient way to just get everything in one fell swoop
#first, get all task types:
ret = []
try:
t = cresource.get("api/task/name/").body_string()
task_names = json.loads(t)
except Exception, ex:
task_names = []
t = {}
logging.error("Error with getting celerymon: %s" % ex)
for tname in task_names:
taskinfo_raw = json.loads(cresource.get('api/task/name/%s' % (tname), params_dict={'limit': task_limit}).body_string())
for traw in taskinfo_raw:
# it's an array of arrays - looping through [<id>, {task_info_dict}]
tinfo = traw[1]
tinfo['name'] = '.'.join(tinfo['name'].split('.')[-2:])
ret.append(tinfo)
ret = sorted(ret, key=lambda x: x['succeeded'], reverse=True)
return HttpResponse(json.dumps(ret), mimetype = 'application/json')
def _get_from_id_or_pk(self, id=None, pk=None, **kwargs):
"""
Returns an object given an id or pk, request directly with the
get_resource_url_detail method without filtering on ids
(as Django's ORM do).
"""
clone = self._clone()
# Instantiation of clone.model is necessary because we can't set
# a staticmethod for get_resource_url_detail and avoid to set it
# for all model without relying on get_resource_url_list
instance = clone.model()
if pk is None:
instance.id = id
else:
instance.pk = pk
extra_args = {}
extra_args.update(kwargs)
extra_args.update(ROA_SSL_ARGS)
resource = Resource(instance.get_resource_url_detail(),
filters=ROA_FILTERS,
**extra_args)
try:
parameters = clone.query.parameters
logger.debug(
u"""Retrieving : "%s" through %s with parameters "%s" """ %
(clone.model.__name__, resource.uri,
force_unicode(parameters)))
response = resource.get(headers=self._get_http_headers(),
**parameters)
except Exception as e:
raise ROAException(e)
response = force_unicode(
response.body_string()).encode(DEFAULT_CHARSET)
for local_name, remote_name in ROA_MODEL_NAME_MAPPING:
response = response.replace(remote_name, local_name)
# Deserializing objects:
data = self.model.get_parser().parse(StringIO(response))
serializer = self.model.get_serializer(data=data)
if not serializer.is_valid():
raise ROAException(
u'Invalid deserialization for {} model: {}'.format(
self.model, serializer.errors))
return serializer.object
def testGetReadingData(self):
"""Test ability to get data associated with a given reading."""
path = "/resources/data/readings/100"
for user in [self.testuser, self.testpoweruser, self.testsuperuser]:
res = Resource(self.baseurl, filters=[user])
output = res.request("GET",
path=path,
params_dict={
"bin_id": 102,
"bin_section_id": 103,
"sensor_type_id": 10
})
self.unauthorizedChecks([None], path)
def _kill_stale_workers():
celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None)
if celery_monitoring:
cresource = Resource(celery_monitoring, timeout=3)
t = cresource.get("api/workers", params_dict={'status': True}).body_string()
all_workers = json.loads(t)
expected_running, expected_stopped = parse_celery_workers(all_workers)
celery = Celery()
celery.config_from_object(settings)
worker_responses = celery.control.ping(timeout=10)
pings = parse_celery_pings(worker_responses)
hosts_to_stop = [hostname for hostname in expected_stopped if hostname in pings]
if hosts_to_stop:
celery.control.broadcast('shutdown', destination=hosts_to_stop)
def check_rabbitmq():
if settings.BROKER_URL.startswith('amqp'):
amqp_parts = settings.BROKER_URL.replace('amqp://', '').split('/')
mq_management_url = amqp_parts[0].replace('5672', '15672')
vhost = amqp_parts[1]
try:
mq = Resource('http://%s' % mq_management_url, timeout=2)
vhost_dict = json.loads(mq.get('api/vhosts', timeout=2).body_string())
for d in vhost_dict:
if d['name'] == vhost:
return ServiceStatus(True, 'RabbitMQ OK')
return ServiceStatus(False, 'RabbitMQ Offline')
except Exception as e:
return ServiceStatus(False, "RabbitMQ Error: %s" % e)
else:
return ServiceStatus(False, "RabbitMQ Not configured")
def delete(self):
assert self._get_pk_val() is not None, "%s object can't be deleted " \
"because its %s attribute is set to None." \
% (self._meta.object_name, self._meta.pk.attname)
# Deletion in cascade should be done server side.
resource = Resource(self.get_resource_url_detail(),
filters=ROA_FILTERS,
**ROA_SSL_ARGS)
logger.debug(u"""Deleting : "%s" through %s""" % \
(unicode(self), unicode(resource.uri)))
result = resource.delete(headers={'Content-Type': 'application/json'},
**ROA_CUSTOM_ARGS)
if result.status_int in [200, 202, 204]:
self.pk = None
def check_rabbitmq():
ret = {}
mq_status = "Unknown"
if settings.BROKER_URL.startswith('amqp'):
amqp_parts = settings.BROKER_URL.replace('amqp://', '').split('/')
mq_management_url = amqp_parts[0].replace('5672', '15672')
vhost = amqp_parts[1]
try:
mq = Resource('http://%s' % mq_management_url, timeout=2)
vhost_dict = json.loads(
mq.get('api/vhosts', timeout=2).body_string())
mq_status = "Offline"
for d in vhost_dict:
if d['name'] == vhost:
mq_status = 'RabbitMQ OK'
except Exception, ex:
mq_status = "RabbitMQ Error: %s" % ex
def check_heartbeat():
celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None)
if celery_monitoring:
cresource = Resource(celery_monitoring, timeout=3)
t = cresource.get("api/workers", params_dict={
'status': True
}).body_string()
all_workers = json.loads(t)
bad_workers = []
for hostname, status in all_workers.items():
if not status:
bad_workers.append('* {} celery worker down'.format(hostname))
if bad_workers:
return ServiceStatus(False, '\n'.join(bad_workers))
is_alive = heartbeat.is_alive()
return ServiceStatus(is_alive, "OK" if is_alive else "DOWN")
def testAccountsNewDelete(self):
path = "/resources/accounts"
user = self.testpoweruser
res = Resource(self.baseurl, filters=[user])
email = "*****@*****.**"
self.assertEqual(None, util.getAccountByEmail(email),
"Account to add already exists.")
params = {
"name": "New User Test",
"email": "*****@*****.**",
"phone": "15555555555",
"privilege_id": 100
}
output = res.request("POST", path=path, params_dict=params)
self.assertEqual(200, output.status_int,
"Wrong response code from post.")
xlink = json.loads(output.body_string())
#check account got created.
new_account = util.getAccountByEmail(email)
self.assertNotEqual(None, new_account, "New account was not created")
#Check link returned is correct.
path = xlink["xlink"][0]
self.assertEquals("/resources/accounts/" + str(new_account.id), path,
"Link returned does not match new account")
#DELETE new account
output = res.request("DELETE", path=path)
self.assertEqual(204, output.status_int,
"Wrong response code from delete.")
new_account = util.getAccountByEmail(email)
self.assertEqual(None, util.getAccountByEmail(email),
"Failed to delete account.")
#authorized checks
#new
self.unauthorizedChecks(
[None, self.testuser, self.testfilluser, self.testconfiguser],
"/resources/accounts",
"POST",
params_dict=params)
#delete
self.unauthorizedChecks(
[None, self.testuser, self.testfilluser, self.testconfiguser],
"/resources/accounts/1", "DELETE")
def testSensorsGetUpdate(self):
xlink = self.newSensor()
id = self.getFirstIdFromXLink(xlink)
self.convert_py = "8 + 20"
self.newparams['convert_py'] = self.convert_py
path = "/resources/conf/sensors/" + str(id)
for user in [self.testconfiguser, self.testpoweruser, self.testsuperuser]:
self.convert_py = "5 +" + str(random.randrange(100, 999))
self.newparams['convert_py'] = self.convert_py
res = Resource(self.baseurl, filters=[user])
output = res.request("PUT", path=path, params_dict=self.newparams)
self.assertEqual(204, output.status_int, "Wrong response code: " + user.credentials[0])
sensor = self.getSensor(id)
self.assertEqual(self.convert_py, sensor['convert_py'], 'Sensor was not updated.')
self.unauthorizedChecks([None, self.testuser, self.testfilluser], path, method='PUT', params_dict=self.newparams)
self.unauthorizedChecks([None], path, method='GET')
self.deleteSensor(id)
def testDevicesGetUpdate(self):
xlink = self.newDevice()
id = self.getFirstIdFromXLink(xlink)
self.name = "updated device"
self.newparams['name'] = self.name
path = "/resources/conf/devices/" + str(id)
for user in [self.testconfiguser, self.testpoweruser, self.testsuperuser]:
self.name = "test device" + str(random.randrange(100, 999))
self.newparams['name'] = self.name
res = Resource(self.baseurl, filters=[user])
output = res.request("PUT", path=path, params_dict=self.newparams)
self.assertEqual(204, output.status_int, "Wrong response code: " + user.credentials[0])
device = self.getDevice(id)
self.assertEqual(self.name, device['name'], 'Device was not updated.')
self.unauthorizedChecks([None, self.testuser, self.testfilluser], path, method='PUT', params_dict=self.newparams)
self.unauthorizedChecks([None], path, method='GET')
self.deleteDevice(id)
def test_custom_serializer(self):
register_serializer('custom', 'examples.django_roa_client.serializers')
initial_roa_format_setting = settings.ROA_FORMAT
settings.ROA_FORMAT = 'custom'
page = RemotePage.objects.create(title=u'A custom serialized page')
self.assertEqual(repr(page),
'<RemotePage: A custom serialized page (1)>')
r = Resource('http://127.0.0.1:8081/django_roa_server/remotepage/',
filters=ROA_FILTERS)
response = r.get(**{'format': 'custom'})
self.assertEqual(
repr(response.body_string()),
'\'<?xml version="1.0" encoding="utf-8"?>\\n<django-test version="1.0">\\n <object pk="1" model="django_roa_server.remotepage">\\n <field type="CharField" name="title">A custom serialized page</field>\\n </object>\\n</django-test>\''
)
self.assertEqual(len(RemotePage.objects.all()), 1)
page = RemotePage.objects.get(id=page.id)
self.assertEqual(repr(page),
'<RemotePage: A custom serialized page (1)>')
settings.ROA_FORMAT = initial_roa_format_setting
def __init__(self,
MIDpass,
configPath,
baseURL,
uploadPath,
RCstatusPath,
MIDname=None):
# self.baseURL = defaultConfig.baseURL
# self.password = defaultConfig
# obtain base configuration
self.MIDpass = MIDpass
self.baseURL = baseURL
self.configPath = configPath
self.resource = Resource(baseURL, timeout=60)
# self.upResource = Resource("http://isadoredev1.exotericanalytics.com:5050")
self.uploadPath = uploadPath
self.RCstatusPath = RCstatusPath
self.MIDname = MIDname
def __init__(self,
url=GITHUB_URL,
params=None,
payload=None,
headers=None,
filters=None,
access_token=None):
self.url = url
self.params = params or dict()
self.payload = payload or dict()
self.headers = headers or {'Content-Type': 'application/json'}
filters = filters or list()
self.resource = Resource(
url,
pool=ConnectionPool(factory=Connection),
filters=filters,
)
if access_token is not None:
self.params["access_token"] = access_token
