def system_info(request): def human_bytes(bytes): #source: https://github.com/bartTC/django-memcache-status bytes = float(bytes) if bytes >= 1073741824: gigabytes = bytes / 1073741824 size = '%.2fGB' % gigabytes elif bytes >= 1048576: megabytes = bytes / 1048576 size = '%.2fMB' % megabytes elif bytes >= 1024: kilobytes = bytes / 1024 size = '%.2fKB' % kilobytes else: size = '%.2fB' % bytes return size context = get_hqadmin_base_context(request) context['couch_update'] = request.GET.get('couch_update', 5000) context['celery_update'] = request.GET.get('celery_update', 10000) context['hide_filters'] = True if hasattr(os, 'uname'): context['current_system'] = os.uname()[1] #from dimagi.utils import gitinfo #context['current_ref'] = gitinfo.get_project_info() #removing until the async library is updated context['current_ref'] = {} if settings.COUCH_USERNAME == '' and settings.COUCH_PASSWORD == '': couchlog_resource = Resource("http://%s/" % (settings.COUCH_SERVER_ROOT)) else: couchlog_resource = Resource("http://%s:%s@%s/" % (settings.COUCH_USERNAME, settings.COUCH_PASSWORD, settings.COUCH_SERVER_ROOT)) try: #todo, fix on bigcouch/cloudant context['couch_log'] = "Will be back online shortly" if is_bigcouch() \ else couchlog_resource.get('_log', params_dict={'bytes': 2000 }).body_string() except Exception, ex: context['couch_log'] = "unable to open couch log: %s" % ex
def _delete_page(self, path): """delete page at a given path""" retval = None res = Resource(self.host) page = res.delete(path) data = page.body_string() if data: retval = json.loads(data) return retval
def __init__(self, bot): IPlugin.__init__(self, bot) self.data = None self.config = None self.url = None self.ignored = [] self.auth = None self.project_id = None self.resource = None self.lastCheck = 0 self.config = globalConfig.get('plugins.redmine') if self.config is None: logging.error('Redmine: Disabled.') return self.data = { 'last-bug-created': 0, 'ignored-names': [ '/^Not\-[0-9]+/' # Notifico bots ] } self.LoadPluginData() self.url = globalConfig.get('plugins.redmine.url', None) if self.url is None: logging.error('Redmine: Disabled.') return self.ignored = [] for ignoretok in self.data.get('ignored-names', ['/^Not\-[0-9]/']): if ignoretok.startwith('/') and ignoretok.endwith('/'): self.ignored += [re.compile(ignoretok[1:-1])] else: self.ignored += [re.compile('^' + re.escape(ignoretok) + '$')] self.auth = BasicAuth(globalConfig.get('plugins.redmine.apikey', None), str(random.random())) self.project_id = globalConfig.get('plugins.redmine.project', None) if self.project_id is None: logging.warning('Redmine: Not going to check for bug updates.') self.bug_info_format = globalConfig.get( 'plugins.redmine.bug-info-format', 'Redmine #{ID} - {AUTHOR} - {STATUS} - {SUBJECT}{CRLF}{URL}') self.new_bug_format = globalConfig.get( 'plugins.redmine.new-bug-format', 'NEW ISSUE: {URL} (#{ID}: {SUBJECT})') self.resource = Resource(self.url, filters=[self.auth]) self.bug_regex = re.compile(r'#(\d+)\b') self.lastCheck = 0
def getJSON(self, url): resource = Resource(url, filters=[auth]) response = resource.get(headers={'Content-Type': 'application/json'}) if response.status_int == 200: # Not all resources will return 200 on success. There are other success status codes. Like 204. We've read # the documentation for though and know what to expect here. versions = json.loads(response.body_string()) return versions else: print response.status_int # print response. return None
def unauthorizedChecks(self, users, path, method="GET", params_dict=None): """:param users: list of BasicAuths to try, expecting Unauthorized""" for user in users: name = "None" if (user): name = user.credentials[0] exception = False try: res = Resource(self.baseurl, filters=[user]) output = res.request(method, path, params_dict=params_dict) except Unauthorized: exception = True self.assertTrue(exception, "No unauthorized exception for " + name)
def testGeneralUpdate(self): path = "/resources/conf/general" for user in [self.testconfiguser, self.testpoweruser, self.testsuperuser]: newinterval = random.randrange(300, 600) general = self.getJSONResource(path, user) general['interval'] = newinterval res = Resource(self.baseurl, filters=[user]) output = res.request("PUT", path=path, params_dict=general) self.assertEqual(204, output.status_int, "Wrong response code: " + user.credentials[0]) general = self.getJSONResource(path, user) self.assertEquals(newinterval, int(general['interval']), "General not updated.") self.unauthorizedChecks([None, self.testuser, self.testfilluser], path, method="PUT")
def is_pullrequest_tested(pr,token,owner,repo): url='https://api.github.com/repos/'+owner+'/'+repo+'/pulls/'+pr+'/files' resource = Resource(url, pool=pool) headers = {'Content-Type' : 'application/json' } headers['Authorization'] = 'token %s' % token response = resource.get(headers = headers) files = json.loads(response.body_string()) #print 'NORMAL:', json.dumps(files, sort_keys=True,indent=2) for x in files: if 'test' in x['filename'].lower(): return True; return False;
def __init__(self, irc): self.__parent = super(Redmine, self) self.__parent.__init__(irc) self.saidBugs = ircutils.IrcDict() sayTimeout = self.registryValue('bugSnarferTimeout') for k in irc.state.channels.keys(): self.saidBugs[k] = TimeoutQueue(sayTimeout) self.url = self.registryValue('urlbase') self.auth = BasicAuth(self.registryValue('apikey'), str(random.random())) self.resource = Resource(self.url, filters=[self.auth])
def get_issue(key): """ Given an issue key (i.e. JRA-9) return the JSON representation of it. This is the only place where we deal with JIRA's REST API. """ print('Fetching ' + key) # we need to expand subtasks and links since that's what we care about here. resource = Resource(url + ('/rest/api/latest/issue/%s' % key), pool_instance=pool, filters=[auth]) response = resource.get(headers = {'Content-Type' : 'application/json'}) if response.status_int == 200: # Not all resources will return 200 on success. There are other success status codes. Like 204. We've read # the documentation for though and know what to expect here. issue = json.loads(response.body_string()) return issue else: return None
def testUpdateByUserForSelf(self): """succeed in updating alarm belonging to self""" path = "/resources/alarms/115" # id taken from DB user = self.testuser res = Resource(self.baseurl, filters=[user]) # perform request output = res.request("PUT",path=path, params_dict = {"alarm_type_id": 14, # changeing from 2 to 1 "account_id": 104, # std user from the DB "greater_than_p": True, "value": 105}) # check that update succeeded self.assertEqual(202, output.status_int, "User was unable to update alarm for self.")
def testDeleteNonExistantAlarm(self): """Power user to fail in deleting non-existant alarm""" path = "/resources/alarms/8" user = self.testpoweruser res = Resource(self.baseurl, filters=[user]) try: exception = False output = res.request("DELETE",path=path) except: # which exception should I catch? exception = True self.assertTrue(exception,"Failed to return 404 for missing alarm.")
def iterator(self): """ An iterator over the results from applying this QuerySet to the remote web service. """ resource = Resource(self.model.get_resource_url_list(), filters=ROA_FILTERS, **ROA_SSL_ARGS) try: parameters = self.query.parameters logger.debug( u"""Requesting: "%s" through %s with parameters "%s" """ % (self.model.__name__, resource.uri, force_unicode(parameters))) response = resource.get(headers=self._get_http_headers(), **parameters) except ResourceNotFound: return except Exception as e: raise ROAException(e) response = force_unicode( response.body_string()).encode(DEFAULT_CHARSET) # Deserializing objects: data = self.model.get_parser().parse(StringIO(response)) # Check limit_start and limit_stop arguments for pagination and only # slice data if they are both numeric and there are results left to go. # We only perform this check on lists. limit_start = getattr(self.query, 'limit_start', None) limit_stop = getattr(self.query, 'limit_stop', None) if (isinstance(limit_start, int) and isinstance(limit_stop, int) and limit_stop - limit_start < len(data) and limit_stop <= len(data) and isinstance(data, list)): data = data[limit_start:limit_stop] # [] is the case of empty no-paginated result if data != []: serializer = self.model.get_serializer(data=data) if not serializer.is_valid(): raise ROAException( u'Invalid deserialization for %s model: %s' % (self.model, serializer.errors)) i = 0 for obj in serializer.validated_data: obj['id'] = serializer.initial_data[i].get('id', None) i += 1 yield self.model(**obj)
def _delete_page(host, path): """delete page at a given path""" retval = None if "http://" not in host: host = "http://{0}".format(host) res = Resource(host) page = res.delete(path) data = page.body_string() if data: retval = json.loads(data) return retval
def testGet(self): """Test ability to get list of alarm events""" path = "/resources/alarms/events" for user in [self.testuser, self.testpoweruser, self.testsuperuser]: # perform request res = Resource(self.baseurl, filters=[user]) output = res.request("GET",path=path, params_dict = {"begin": "010111134553", "end": "120111134553"}) self.assertEqual(200,output.status_int, "User, " + str(user) + " was unable to obtain list of alarm events.") self.unauthorizedChecks([None], path)
def testDeleteByUserForOthers(self): """fail to delete other's alarm contact""" path = "/resources/alarms/alarm_contacts/100" # alarm belonging to a poweruser user = self.testuser res = Resource(self.baseurl, filters=[user]) try: exception = False output = res.request("DELETE",path=path) except: # which exception should I catch? exception = True self.assertTrue(exception,"Failed to return 404 for missing alarm.")
def testNewByUserForSelf(self): """create alarm contact for self""" path = "/resources/alarms" user = self.testuser res = Resource(self.baseurl, filters=[user]) # perform request output = res.request("POST",path=path, params_dict = {"alarm_type_id": 10, "account_id":104, "alarm_contact_type_ids": 100}) # check if addition failed self.assertEqual(200, output.status_int, "User was unable to add alarm contact for self.") # get the returned url xlink = json.loads(output.body_string())
def system_ajax(request): """ Utility ajax functions for polling couch and celerymon """ type = request.GET.get('api', None) task_limit = getattr(settings, 'CELERYMON_TASK_LIMIT', 5) celerymon_url = getattr(settings, 'CELERYMON_URL', '') db = XFormInstance.get_db() ret = {} if type == "_active_tasks": tasks = [] if is_bigcouch() else filter(lambda x: x['type'] == "indexer", db.server.active_tasks()) #for reference structure is: # tasks = [{'type': 'indexer', 'pid': 'foo', 'database': 'mock', # 'design_document': 'mockymock', 'progress': 0, # 'started_on': 1349906040.723517, 'updated_on': 1349905800.679458, # 'total_changes': 1023}, # {'type': 'indexer', 'pid': 'foo', 'database': 'mock', # 'design_document': 'mockymock', 'progress': 70, # 'started_on': 1349906040.723517, 'updated_on': 1349905800.679458, # 'total_changes': 1023}] return HttpResponse(json.dumps(tasks), mimetype='application/json') elif type == "_stats": return HttpResponse(json.dumps({}), mimetype = 'application/json') elif type == "_logs": pass if celerymon_url != '': cresource = Resource(celerymon_url, timeout=3) if type == "celerymon_poll": #inefficient way to just get everything in one fell swoop #first, get all task types: ret = [] try: t = cresource.get("api/task/name/").body_string() task_names = json.loads(t) except Exception, ex: task_names = [] t = {} logging.error("Error with getting celerymon: %s" % ex) for tname in task_names: taskinfo_raw = json.loads(cresource.get('api/task/name/%s' % (tname), params_dict={'limit': task_limit}).body_string()) for traw in taskinfo_raw: # it's an array of arrays - looping through [<id>, {task_info_dict}] tinfo = traw[1] tinfo['name'] = '.'.join(tinfo['name'].split('.')[-2:]) ret.append(tinfo) ret = sorted(ret, key=lambda x: x['succeeded'], reverse=True) return HttpResponse(json.dumps(ret), mimetype = 'application/json')
def _get_from_id_or_pk(self, id=None, pk=None, **kwargs): """ Returns an object given an id or pk, request directly with the get_resource_url_detail method without filtering on ids (as Django's ORM do). """ clone = self._clone() # Instantiation of clone.model is necessary because we can't set # a staticmethod for get_resource_url_detail and avoid to set it # for all model without relying on get_resource_url_list instance = clone.model() if pk is None: instance.id = id else: instance.pk = pk extra_args = {} extra_args.update(kwargs) extra_args.update(ROA_SSL_ARGS) resource = Resource(instance.get_resource_url_detail(), filters=ROA_FILTERS, **extra_args) try: parameters = clone.query.parameters logger.debug( u"""Retrieving : "%s" through %s with parameters "%s" """ % (clone.model.__name__, resource.uri, force_unicode(parameters))) response = resource.get(headers=self._get_http_headers(), **parameters) except Exception as e: raise ROAException(e) response = force_unicode( response.body_string()).encode(DEFAULT_CHARSET) for local_name, remote_name in ROA_MODEL_NAME_MAPPING: response = response.replace(remote_name, local_name) # Deserializing objects: data = self.model.get_parser().parse(StringIO(response)) serializer = self.model.get_serializer(data=data) if not serializer.is_valid(): raise ROAException( u'Invalid deserialization for {} model: {}'.format( self.model, serializer.errors)) return serializer.object
def testGetReadingData(self): """Test ability to get data associated with a given reading.""" path = "/resources/data/readings/100" for user in [self.testuser, self.testpoweruser, self.testsuperuser]: res = Resource(self.baseurl, filters=[user]) output = res.request("GET", path=path, params_dict={ "bin_id": 102, "bin_section_id": 103, "sensor_type_id": 10 }) self.unauthorizedChecks([None], path)
def _kill_stale_workers(): celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None) if celery_monitoring: cresource = Resource(celery_monitoring, timeout=3) t = cresource.get("api/workers", params_dict={'status': True}).body_string() all_workers = json.loads(t) expected_running, expected_stopped = parse_celery_workers(all_workers) celery = Celery() celery.config_from_object(settings) worker_responses = celery.control.ping(timeout=10) pings = parse_celery_pings(worker_responses) hosts_to_stop = [hostname for hostname in expected_stopped if hostname in pings] if hosts_to_stop: celery.control.broadcast('shutdown', destination=hosts_to_stop)
def check_rabbitmq(): if settings.BROKER_URL.startswith('amqp'): amqp_parts = settings.BROKER_URL.replace('amqp://', '').split('/') mq_management_url = amqp_parts[0].replace('5672', '15672') vhost = amqp_parts[1] try: mq = Resource('http://%s' % mq_management_url, timeout=2) vhost_dict = json.loads(mq.get('api/vhosts', timeout=2).body_string()) for d in vhost_dict: if d['name'] == vhost: return ServiceStatus(True, 'RabbitMQ OK') return ServiceStatus(False, 'RabbitMQ Offline') except Exception as e: return ServiceStatus(False, "RabbitMQ Error: %s" % e) else: return ServiceStatus(False, "RabbitMQ Not configured")
def delete(self): assert self._get_pk_val() is not None, "%s object can't be deleted " \ "because its %s attribute is set to None." \ % (self._meta.object_name, self._meta.pk.attname) # Deletion in cascade should be done server side. resource = Resource(self.get_resource_url_detail(), filters=ROA_FILTERS, **ROA_SSL_ARGS) logger.debug(u"""Deleting : "%s" through %s""" % \ (unicode(self), unicode(resource.uri))) result = resource.delete(headers={'Content-Type': 'application/json'}, **ROA_CUSTOM_ARGS) if result.status_int in [200, 202, 204]: self.pk = None
def check_rabbitmq(): ret = {} mq_status = "Unknown" if settings.BROKER_URL.startswith('amqp'): amqp_parts = settings.BROKER_URL.replace('amqp://', '').split('/') mq_management_url = amqp_parts[0].replace('5672', '15672') vhost = amqp_parts[1] try: mq = Resource('http://%s' % mq_management_url, timeout=2) vhost_dict = json.loads( mq.get('api/vhosts', timeout=2).body_string()) mq_status = "Offline" for d in vhost_dict: if d['name'] == vhost: mq_status = 'RabbitMQ OK' except Exception, ex: mq_status = "RabbitMQ Error: %s" % ex
def check_heartbeat(): celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None) if celery_monitoring: cresource = Resource(celery_monitoring, timeout=3) t = cresource.get("api/workers", params_dict={ 'status': True }).body_string() all_workers = json.loads(t) bad_workers = [] for hostname, status in all_workers.items(): if not status: bad_workers.append('* {} celery worker down'.format(hostname)) if bad_workers: return ServiceStatus(False, '\n'.join(bad_workers)) is_alive = heartbeat.is_alive() return ServiceStatus(is_alive, "OK" if is_alive else "DOWN")
def testAccountsNewDelete(self): path = "/resources/accounts" user = self.testpoweruser res = Resource(self.baseurl, filters=[user]) email = "*****@*****.**" self.assertEqual(None, util.getAccountByEmail(email), "Account to add already exists.") params = { "name": "New User Test", "email": "*****@*****.**", "phone": "15555555555", "privilege_id": 100 } output = res.request("POST", path=path, params_dict=params) self.assertEqual(200, output.status_int, "Wrong response code from post.") xlink = json.loads(output.body_string()) #check account got created. new_account = util.getAccountByEmail(email) self.assertNotEqual(None, new_account, "New account was not created") #Check link returned is correct. path = xlink["xlink"][0] self.assertEquals("/resources/accounts/" + str(new_account.id), path, "Link returned does not match new account") #DELETE new account output = res.request("DELETE", path=path) self.assertEqual(204, output.status_int, "Wrong response code from delete.") new_account = util.getAccountByEmail(email) self.assertEqual(None, util.getAccountByEmail(email), "Failed to delete account.") #authorized checks #new self.unauthorizedChecks( [None, self.testuser, self.testfilluser, self.testconfiguser], "/resources/accounts", "POST", params_dict=params) #delete self.unauthorizedChecks( [None, self.testuser, self.testfilluser, self.testconfiguser], "/resources/accounts/1", "DELETE")
def testSensorsGetUpdate(self): xlink = self.newSensor() id = self.getFirstIdFromXLink(xlink) self.convert_py = "8 + 20" self.newparams['convert_py'] = self.convert_py path = "/resources/conf/sensors/" + str(id) for user in [self.testconfiguser, self.testpoweruser, self.testsuperuser]: self.convert_py = "5 +" + str(random.randrange(100, 999)) self.newparams['convert_py'] = self.convert_py res = Resource(self.baseurl, filters=[user]) output = res.request("PUT", path=path, params_dict=self.newparams) self.assertEqual(204, output.status_int, "Wrong response code: " + user.credentials[0]) sensor = self.getSensor(id) self.assertEqual(self.convert_py, sensor['convert_py'], 'Sensor was not updated.') self.unauthorizedChecks([None, self.testuser, self.testfilluser], path, method='PUT', params_dict=self.newparams) self.unauthorizedChecks([None], path, method='GET') self.deleteSensor(id)
def testDevicesGetUpdate(self): xlink = self.newDevice() id = self.getFirstIdFromXLink(xlink) self.name = "updated device" self.newparams['name'] = self.name path = "/resources/conf/devices/" + str(id) for user in [self.testconfiguser, self.testpoweruser, self.testsuperuser]: self.name = "test device" + str(random.randrange(100, 999)) self.newparams['name'] = self.name res = Resource(self.baseurl, filters=[user]) output = res.request("PUT", path=path, params_dict=self.newparams) self.assertEqual(204, output.status_int, "Wrong response code: " + user.credentials[0]) device = self.getDevice(id) self.assertEqual(self.name, device['name'], 'Device was not updated.') self.unauthorizedChecks([None, self.testuser, self.testfilluser], path, method='PUT', params_dict=self.newparams) self.unauthorizedChecks([None], path, method='GET') self.deleteDevice(id)
def test_custom_serializer(self): register_serializer('custom', 'examples.django_roa_client.serializers') initial_roa_format_setting = settings.ROA_FORMAT settings.ROA_FORMAT = 'custom' page = RemotePage.objects.create(title=u'A custom serialized page') self.assertEqual(repr(page), '<RemotePage: A custom serialized page (1)>') r = Resource('http://127.0.0.1:8081/django_roa_server/remotepage/', filters=ROA_FILTERS) response = r.get(**{'format': 'custom'}) self.assertEqual( repr(response.body_string()), '\'<?xml version="1.0" encoding="utf-8"?>\\n<django-test version="1.0">\\n <object pk="1" model="django_roa_server.remotepage">\\n <field type="CharField" name="title">A custom serialized page</field>\\n </object>\\n</django-test>\'' ) self.assertEqual(len(RemotePage.objects.all()), 1) page = RemotePage.objects.get(id=page.id) self.assertEqual(repr(page), '<RemotePage: A custom serialized page (1)>') settings.ROA_FORMAT = initial_roa_format_setting
def __init__(self, MIDpass, configPath, baseURL, uploadPath, RCstatusPath, MIDname=None): # self.baseURL = defaultConfig.baseURL # self.password = defaultConfig # obtain base configuration self.MIDpass = MIDpass self.baseURL = baseURL self.configPath = configPath self.resource = Resource(baseURL, timeout=60) # self.upResource = Resource("http://isadoredev1.exotericanalytics.com:5050") self.uploadPath = uploadPath self.RCstatusPath = RCstatusPath self.MIDname = MIDname
def __init__(self, url=GITHUB_URL, params=None, payload=None, headers=None, filters=None, access_token=None): self.url = url self.params = params or dict() self.payload = payload or dict() self.headers = headers or {'Content-Type': 'application/json'} filters = filters or list() self.resource = Resource( url, pool=ConnectionPool(factory=Connection), filters=filters, ) if access_token is not None: self.params["access_token"] = access_token