def asyncresults(self, asyncjob,api_key,secret_key): cs_api = CloudstackAPI(api_key=api_key, secret_key=secret_key) qryasyncjob = cs_api.request(dict({'command':'queryAsyncJobResult', 'jobid':asyncjob})) qryasyncjob = dotdictify(qryasyncjob) qryasyncjob = qryasyncjob.queryasyncjobresultresponse for key, value in qryasyncjob.items(): if key == 'jobstatus': if value == 0: status=0 while status==0: qryasyncjob = cs_api.request(dict({'command':'queryAsyncJobResult', 'jobid':asyncjob})) qryasyncjob = dotdictify(qryasyncjob) qryasyncjob = qryasyncjob.queryasyncjobresultresponse for key, value in qryasyncjob.iteritems(): if key == 'jobstatus' : status = value if key == 'jobstatus' : CloudstackAPI.jobresults = value time.sleep(5) queryasyncjobresponse = cs_api.request(dict({'command':'queryAsyncJobResult', 'jobid':asyncjob})) return queryasyncjobresponse
def __get_all_paged_entities(self, path, args): logger.info("Fetching data from paged url: %s", path) url = os.environ.get("base_url") + path access_token = get_token() next_page = url page_counter = 1 while next_page is not None: if os.environ.get('sleep') is not None: logger.info("sleeping for %s milliseconds", os.environ.get('sleep') ) sleep(float(os.environ.get('sleep'))) logger.info("Fetching data from url: %s", next_page) if "$skiptoken" not in next_page: req = requests.get(next_page, params=args, headers={"Authorization": "Bearer " + access_token}) else: req = requests.get(next_page, headers={"Authorization": "Bearer " + access_token}) if req.status_code != 200: logger.error("Unexpected response status code: %d with response text %s" % (req.status_code, req.text)) raise AssertionError ("Unexpected response status code: %d with response text %s"%(req.status_code, req.text)) res = dotdictify.dotdictify(json.loads(req.text)) for entity in res.get(os.environ.get("entities_path")): yield(entity) if res.get(os.environ.get('next_page')) is not None: page_counter+=1 next_page = res.get(os.environ.get('next_page')) else: next_page = None logger.info('Returning entities from %i pages', page_counter)
def __get_all_entities(self, path): logger.info("Fetching data from url: %s", path) token = get_token(path) headers = { 'Accept': 'application/json', 'content_type': 'application/json' } url = os.environ.get('get_url') + "?access_token=" + token req = requests.get(url, headers=headers) if req.status_code != 200: logger.error( "Unexpected response status code: %d with response text %s" % (req.status_code, req.text)) raise AssertionError( "Unexpected response status code: %d with response text %s" % (req.status_code, req.text)) res = dotdictify.dotdictify(json.loads(req.text)) if path == "user": for entity in res.get(os.environ.get("entities_path_user")): yield (entity) if path == "organization": for entity in res.get(os.environ.get("entities_path_org")): yield (entity) else: logger.info("method not recognized") logger.info('Returning entities from %s', path)
def get_token(path): headers = {} logger.info("Creating header") if path == "user": headers = { "client_id": os.environ.get('client_id_user'), "client_secret": os.environ.get('client_secret_user'), "grant_type": os.environ.get('grant_type') } elif path == "organization": headers = { "client_id": os.environ.get('client_id_org'), "client_secret": os.environ.get('client_secret_org'), "grant_type": os.environ.get('grant_type') } elif path == "post_user": headers = { "client_id": os.environ.get('client_id_post'), "client_secret": os.environ.get('client_secret_post'), "grant_type": os.environ.get('grant_type') } else: logger.info("undefined method") sys.exit() resp = requests.get(url=os.environ.get('token_url'), headers=headers).json() token = dotdictify.dotdictify(resp).response.responseMessage.access_token logger.info("Received access token from " + os.environ.get('token_url')) return token
def __get_all_paged_entities(self, path, url_parameters): logger.info("Fetching data from paged url: %s", path) url = os.environ.get("baseurl") + path url = call_url(url, url_parameters, url_parameters.get(os.environ.get('startpage'))) has_more_results = True page_counter = 1 while has_more_results: if os.environ.get('sleep') is not None: logger.info("sleeping for %s milliseconds", os.environ.get('sleep')) sleep(float(os.environ.get('sleep'))) logger.info("Fetching data from url: %s", url) req = requests.get(url, headers=headers) if req.status_code != 200: logger.error( "Unexpected response status code: %d with response text %s" % (req.status_code, req.text)) raise AssertionError( "Unexpected response status code: %d with response text %s" % (req.status_code, req.text)) dict = dotdictify.dotdictify(json.loads(req.text)) for entity in dict.results: yield entity if str_to_bool(dict.get(os.environ.get('next_page_path'))): page_counter += 1 url = os.environ.get("baseurl") + path url = call_url(url, url_parameters, str(page_counter)) else: has_more_results = False logger.info('Returning entities from %i pages', page_counter)
def __get_all_references(self, path): logger.info('Fetching data from paged url: %s', path) url = os.environ.get("base_url") + path headers = { "Accept": "application/json", "Content-Type": "application/json", "Authorization": os.environ.get('token') } reference_data = json.loads( os.environ.get('reference_post').replace("'", "\"")) total_amount = json.loads( requests.post(url, data=json.dumps(reference_data), headers=headers).text)["total"] counter = 0 size = 10 while counter < total_amount: req = requests.post(url, data=json.dumps(reference_data), headers=headers) res = dotdictify.dotdictify(json.loads(req.text)) counter += size reference_data["offset"] = counter entities = res.get(os.environ.get("references_path")) for entity in entities: yield (entity.get("reference")) logger.info("returned from all pages")
def __get_all_paged_entities(self, path): logger.info("Fetching data from paged url: %s", path) url = os.environ.get("base_url") + path next_page = url page_counter = 1 while next_page is not None: if os.environ.get('sleep') is not None: logger.info("sleeping for %s milliseconds", os.environ.get('sleep')) sleep(float(os.environ.get('sleep'))) logger.info("Fetching data from url: %s", next_page) req = requests.get(next_page, headers=headers) if req.status_code != 200: logger.error( "Unexpected response status code: %d with response text %s" % (req.status_code, req.text)) raise AssertionError( "Unexpected response status code: %d with response text %s" % (req.status_code, req.text)) dict = dotdictify.dotdictify(json.loads(req.text)) for entity in dict.get(os.environ.get("entities_path")): yield transform(entity) if dict.get(os.environ.get('next_page')) is not None: page_counter += 1 next_page = dict.get(os.environ.get('next_page')) else: next_page = None logger.info('Returning entities from %i pages', page_counter)
def __get_all_siteurls(self, posted_entities): logger.info('fetching site urls') access_token = get_token() for entity in posted_entities: url = "https://graph.microsoft.com/v1.0/groups/" + set_group_id(entity) + "/sites/root" req = requests.get(url=url, headers={"Authorization": "Bearer " + access_token}) if req.status_code != 200: logger.info('no url') else: res = dotdictify.dotdictify(json.loads(req.text)) res['_id'] = set_group_id(entity) yield res
def get_token(): logger.info("Creating header") headers= {} payload = { "client_id":os.environ.get('client_id'), "client_secret":os.environ.get('client_secret'), "grant_type": os.environ.get('grant_type'), "resource": os.environ.get('resource') } #logger.info(payload) resp = requests.post(url=os.environ.get('token_url'), data=payload, headers=headers).json() token = dotdictify.dotdictify(resp).access_token logger.info("Received access token from " + os.environ.get('token_url')) return token
def transform(obj): res = {} for k, v in obj.items(): if k == "image": if dotdictify.dotdictify(v).large.url is not None: res[k] = encode(v) else: pass try: _ = json.dumps(v) except Exception: pass else: res[k] = v return res
def transform(obj): res = {} for k, v in obj.items(): if k == "image": if dotdictify.dotdictify(v).large.url is not None: logger.info("Encoding images from url to base64...") res[k] = encode(v) else: pass try: _ = json.dumps(v) except Exception: pass else: res[k] = v return res
def __get_all_paged_entities(self, path): logger.info("Fetching data from paged url: %s", path) url = os.environ.get("base_url") + path next_page = url page_counter = 1 while next_page is not None: logger.info("Fetching data from url: %s", next_page) req = requests.get(next_page, headers=headers) if req.status_code != 200: req = self.check_error(req, next_page, headers, 'get') dict = dotdictify.dotdictify(json.loads(req.text)) for entity in dict.get(os.environ.get("entities_path")): yield transform(entity) if dict.get(os.environ.get('next_page')) is not None: page_counter += 1 next_page = dict.get(os.environ.get('next_page')) else: next_page = None logger.info('Returning entities from %i pages', page_counter)
def __get_all_paged_entities(self, path, args): logger.info("Fetching data from url: %s", path) url = os.environ.get("base_url") + path req = requests.get(url, headers={ "Accept": "Application/json", "Authorization": "Basic " + os.environ.get('basic_token') }) if req.status_code != 200: logger.error( "Unexpected response status code: %d with response text %s" % (req.status_code, req.text)) raise AssertionError( "Unexpected response status code: %d with response text %s" % (req.status_code, req.text)) res = dotdictify.dotdictify(json.loads(req.text)) for entity in res['value']: yield (entity) logger.info('Returning entities')
return data def sub_dict(somedict, somekey, default = None): return dict([ (somekey, somedict.get(somekey, default)) ]) dict_out ={} dictofsubs ={} data_dict = list_subs() dict_out = sub_dict(data_dict, 'results') dictofsubs = sub_dict(dict_out, 'id') print "%%%%%%%%%%%%%%%%%%START DOT DICTIFY %%%%%%%%%%%%%%%%%%%%%%%" dotdict= dotdictify.dotdictify(data_dict) print dotdict.totalSize print dotdict.__contains__("results.id") print "#####$$$$$ PRINT SUB DICT OF DICT_OUT & TYPE OF DICT_OUT ^^^^^^^^^^^^^^^^" pprint(dict_out) print type(dict_out) print"$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" print "New Test iteritems of sub dict DICT_OUT" for key, value in dict_out.iteritems(): if key == "id": print "EXTRACT OF DICT INTO DICT"
return data def sub_dict(somedict, somekey, default=None): return dict([(somekey, somedict.get(somekey, default))]) dict_out = {} dictofsubs = {} data_dict = list_subs() dict_out = sub_dict(data_dict, 'results') dictofsubs = sub_dict(dict_out, 'id') print "%%%%%%%%%%%%%%%%%%START DOT DICTIFY %%%%%%%%%%%%%%%%%%%%%%%" dotdict = dotdictify.dotdictify(data_dict) print dotdict.totalSize print dotdict.__contains__("results.id") print "#####$$$$$ PRINT SUB DICT OF DICT_OUT & TYPE OF DICT_OUT ^^^^^^^^^^^^^^^^" pprint(dict_out) print type(dict_out) print "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" print "New Test iteritems of sub dict DICT_OUT" for key, value in dict_out.iteritems(): if key == "id": print "EXTRACT OF DICT INTO DICT"