def select(host,user,pswd,cert=None,format=None,ID=None,template=None,title=None): """Get all data that fits a certain simple query Inputs: host - string, URL of MDCS instance user - string, username of desired account on MDCS server pswd - string, password of desired account on MDCS server cert - string, path to authentication certificate format - string, format of data (can be xml or json) template - string, ID of the schema for particular data ID - string, ID of entry to be retrieved title - string, title of data to be retrieved Output: list of all entries in the database dictionaries each have the keys: title - title of the entry schema - ID of the schema that describes the entry content - content of the entry in either xml or json format _id - ID number of the entry """ url = host.strip("/") + "/rest/explore/select" params = dict() if format: params['dataformat'] = format if ID: params['id'] = ID if template: params['schema'] = template if title: params['title'] = title r = requests.get(url, params=params, auth=(user, pswd), verify=cert) return check_response(r)
def delete(ID,host,user,pswd,cert=None,next=None): url = host.strip("/") + "/rest/types/delete" params = dict() params['id']=ID if next: params['next']=next r = requests.delete(url, params=params, auth=(user, pswd), verify=cert) return check_response(r)
def delete(ID, host, user, pswd, cert=None, next=None): url = host.strip("/") + "/rest/types/delete" params = dict() params['id'] = ID if next: params['next'] = next r = requests.delete(url, params=params, auth=(user, pswd), verify=cert) return check_response(r)
def query(host, user, pswd, cert=None, format=None, query=None, repositories=None): """Query by example. Input: host - string, URL of MDCS instance user - string, username of desired account on MDCS server pswd - string, password of desired account on MDCS server cert - string, path to authentication certificate format - string, format of data (can be xml or json) respositories - string, lists of names of other repositories to be Output: lists where each entry is a dictionary describing entries that match the query. dictionaries each have the keys: title - title of the entry schema - ID of the schema that describes the entry content - content of the entry in either xml or json format _id - ID number of the entry """ url = host.strip("/") + "/rest/explore/query-by-example" data = dict() if format: data['dataformat'] = format if query: data['query'] = query if repositories: data['repositories'] = repositories r = requests.post(url, data=data, auth=(user, pswd), verify=cert) return check_response(r)
def query(host,user,pswd,cert=None,format=None,query=None,repositories=None): """Query by example. Input: host - string, URL of MDCS instance user - string, username of desired account on MDCS server pswd - string, password of desired account on MDCS server cert - string, path to authentication certificate format - string, format of data (can be xml or json) respositories - string, lists of names of other repositories to be Output: lists where each entry is a dictionary describing entries that match the query. dictionaries each have the keys: title - title of the entry schema - ID of the schema that describes the entry content - content of the entry in either xml or json format _id - ID number of the entry """ url = host.strip("/") + "/rest/explore/query-by-example" data = dict() if format: data['dataformat'] = format if query: data['query'] = query if repositories: data['repositories'] = repositories r = requests.post(url, data=data, auth=(user, pswd), verify=cert) return check_response(r)
def select(host, user, pswd, cert=None, format=None, ID=None, template=None, title=None): """Get all data that fits a certain simple query Inputs: host - string, URL of MDCS instance user - string, username of desired account on MDCS server pswd - string, password of desired account on MDCS server cert - string, path to authentication certificate format - string, format of data (can be xml or json) template - string, ID of the schema for particular data ID - string, ID of entry to be retrieved title - string, title of data to be retrieved Output: list of all entries in the database dictionaries each have the keys: title - title of the entry schema - ID of the schema that describes the entry content - content of the entry in either xml or json format _id - ID number of the entry """ url = host.strip("/") + "/rest/explore/select" params = dict() if format: params['dataformat'] = format if ID: params['id'] = ID if template: params['schema'] = template if title: params['title'] = title r = requests.get(url, params=params, auth=(user, pswd), verify=cert) return check_response(r)
def delete_idp(self, idp_id): url = self.host_url + self.api_version + self.federation_string + self.idps_string + idp_id header = self.headers.header_delete header = self._check_header_auth_token(header) resp = requests.delete(url, headers=header, verify=False) if utils.check_response(resp, 204): print("HTTP Status Code: 204\nIdentity provider deleted:") print("\tId: " + idp_id) else: utils.expose_reason(resp)
def delete_group(self, group_name, group_id): url = self.host_url + self.api_version + self.groups_string + group_id header = self.headers.header_delete header = self._check_header_auth_token(header) resp = requests.delete(url, headers=header, verify=False) if utils.check_response(resp, 204): print("HTTP Status Code: 204\nGroup deleted:") print("\tName: " + group_name + "\n\tid: " + group_id) else: utils.expose_reason(resp)
def get_project(self, project_id): url = self.host_url + self.api_version + self.projects_string + project_id header = self.headers.header_get header = self._check_header_auth_token(header) resp = requests.get(url, headers=header, verify=False) if utils.check_response(resp, 200): print("HTTP Status Code: 200\nProject:") print(json.dumps(resp.json(), sort_keys=True, indent=4, separators=(',', ': '))) else: utils.expose_reason(resp)
def get_mapping(self, mapping_id): url = self.host_url + self.api_version + self.federation_string + self.mappings_string + mapping_id header = self.headers.header_get resp = requests.get(url, headers=header, verify=False) if utils.check_response(resp, 200): self.mapping = resp.json().get("mapping") print("HTTP Status Code: 200\nMapping:") print(json.dumps(self.mapping, sort_keys=True, indent=4, separators=(',', ': '))) else: utils.expose_reason(resp)
def list_mappings(self): url = self.host_url + self.api_version + self.federation_string + self.mappings_string header = self.headers.header_get header = self._check_header_auth_token(header) resp = requests.get(url, headers=header, verify=False) if utils.check_response(resp, 200): print("HTTP Status Code: 200\nMappings:") print(json.dumps(resp.json(), sort_keys=True, indent=4, separators=(',', ': '))) else: utils.expose_reason(resp)
def delete_mapping(self, mapping_id): url = self.host_url + self.api_version + self.federation_string + self.mappings_string + mapping_id header = self.headers.header_delete header = self._check_header_auth_token(header) resp = requests.delete(url, headers=header, verify=False) if utils.check_response(resp, 204): print("HTTP Status Code: 204\nMapping deleted:") print("\tId: " + mapping_id) else: utils.expose_reason(resp)
def get_domain(self, domain_id): url = self.host_url + self.api_version + self.domains_string + domain_id header = self.headers.header_get header = self._check_header_auth_token(header) resp = requests.get(url, headers=header, verify=False) if utils.check_response(resp, 200): self.domain = resp.json().get("domain") print("HTTP Status Code: 200\nDomain:") print(json.dumps(self.domain, sort_keys=True, indent=4, separators=(',', ': '))) else: utils.expose_reason(resp)
def select(host,user,pswd,cert=None,ID=None,filename=None,title=None,version=None,typeVersion=None,Hash=None): url = host.strip("/") + "/rest/types/select" params = dict() if ID: params['id']=ID if filename: params['filename']=filename if title: params['title']=title if version: params['version']=version if typeVersion: params['typeVersion']=typeVersion if Hash: params['hash']=Hash r = requests.get(url, params=params, auth=(user, pswd), verify=cert) return check_response(r)
def get_idp(self, idp_id): url = self.host_url + self.api_version + self.federation_string + self.idps_string + idp_id header = self.headers.header_get header = self._check_header_auth_token(header) resp = requests.get(url, headers=header, verify=False) if utils.check_response(resp, 200): self.idp = resp.json().get("identity_provider") print("HTTP Status Code: 200\nIdentity provider:") print(json.dumps(self.idp, sort_keys=True, indent=4, separators=(',', ': '))) else: utils.expose_reason(resp)
def update_group(self, group_id): url = self.host_url + self.api_version + self.groups_string + group_id header = self.headers.header_patch header = self._check_header_auth_token(header) body = self.group resp = requests.patch(url, data=json.dumps(body), headers=header, verify=False) if utils.check_response(resp, 200): self.group = (resp.json().get("group")) print("HTTP Status Code: 200\nGroup updated:") print(json.dumps(self.group, sort_keys=True, indent=4, separators=(',', ': '))) else: utils.expose_reason(resp)
def update_mapping(self, mapping_id): url = self.host_url + self.api_version + self.federation_string + self.mappings_string + mapping_id header = self.headers.header_patch header = self._check_header_auth_token(header) body = self.mapping resp = requests.patch(url, data=json.dumps(body), headers=header, verify=False) if utils.check_response(resp, 200): self.mapping = resp.json().get("mapping") print("HTTP Status Code: 200\nMapping updated:") print(json.dumps(self.mapping, sort_keys=True, indent=4, separators=(',', ': '))) else: utils.expose_reason(resp)
def revoke_role_group_project(self, project_name, project_id, group_name, group_id, role_name, role_id): url = self.host_url + self.api_version + self.projects_string + project_id + self.groups_string + group_id + \ self.roles_string + role_id header = self.headers.header_delete header = self._check_header_auth_token(header) resp = requests.put(url, headers=header, verify=False) if utils.check_response(resp, 204): print("HTTP Status Code: 204\nRevoked role to group on project:") print("Role:\n\tname: " + role_name + "\n\tid: " + role_id) print("Group:\n\tname: " + group_name + "\n\tid: " + group_id) print("Project:\n\tname: " + project_name + "\n\tid: " + project_id) else: utils.expose_reason(resp)
def grant_role_group_domain(self, domain_name, domain_id, group_name, group_id, role_name, role_id): url = self.host_url + self.api_version + self.domains_string + domain_id + self.groups_string + group_id + \ self.roles_string + role_id header = self.headers.header_put header = self._check_header_auth_token(header) resp = requests.put(url, headers=header, verify=False) if utils.check_response(resp, 204): print("HTTP Status Code: 204\nGranted role to group on domain:") print("Role:\n\tname: " + role_name + "\n\tid: " + role_id) print("Domain:\n\tname: " + domain_name + "\n\tid: " + domain_id) print("Group:\n\tname: " + group_name + "\n\tid: " + group_id) else: utils.expose_reason(resp)
def create_idp(self, idp_id): url = self.host_url + self.api_version + self.federation_string + self.idps_string + idp_id header = self.headers.header_post header = self._check_header_auth_token(header) body = self.idp resp = requests.put(url, data=json.dumps(body), headers=header, verify=False) if utils.check_response(resp, 201): self.idp = resp.json().get("identity_provider") print("HTTP Status Code: 201\nIdP created:") print(json.dumps(self.idp, sort_keys=True, indent=4, separators=(',', ': '))) else: utils.expose_reason(resp) self.ids.idp_id = idp_id
def create_domain(self): url = self.host_url + self.api_version + self.domains_string header = self.headers.header_post header = self._check_header_auth_token(header) body = self.domain resp = requests.post(url, data=json.dumps(body), headers=header, verify=False) if utils.check_response(resp, 201): self.domain = (resp.json().get("domain")) print("HTTP Status Code: 201\nDomain created:") print(json.dumps(self.domain, sort_keys=True, indent=4, separators=(',', ': '))) else: utils.expose_reason(resp) self.ids.domain_id = self.domain["id"]
def add(filename,title,host,user,pswd,cert=None,version=None,dependencies=None): url = host.strip("/") + "/rest/types/add" with open(filename, 'r') as f: xsd_data = f.read() data=dict() data['content'] = [xsd_data] data['filename'] = [filename] data['title'] = [title] if version: data['typeVersion'] = [version] if dependencies: data['dependencies[]'] = dependencies r = requests.post(url, data=data, auth=(user, pswd), verify=cert) return check_response(r)
def delete(ID,host,user,pswd,cert=None): """Delete an entry Input: ID - string, ID of object to be deleted host - string, URL of MDCS instance user - string, username of desired account on MDCS server pswd - string, password of desired account on MDCS server cert - string, path to authentication certificate Output: response from MDCS """ url = host.strip("/") + "/rest/explore/delete" params = dict() params['id']=ID r = requests.delete(url, params=params, auth=(user, pswd), verify=cert) return check_response(r)
def delete(ID, host, user, pswd, cert=None): """Delete an entry Input: ID - string, ID of object to be deleted host - string, URL of MDCS instance user - string, username of desired account on MDCS server pswd - string, password of desired account on MDCS server cert - string, path to authentication certificate Output: response from MDCS """ url = host.strip("/") + "/rest/explore/delete" params = dict() params['id'] = ID r = requests.delete(url, params=params, auth=(user, pswd), verify=cert) return check_response(r)
def curate(file_name, file_title, template_id, host, user, pswd, cert=None, content=None): if content is None: with open(file_name, 'r') as f: content = f.read() data = dict() data['content'] = [content] data['schema'] = [template_id] data['title'] = [file_title] url = host.strip("/") + "/rest/curate" r = requests.post(url, data=data, auth=(user, pswd), verify=cert) return check_response(r)
def select(host, user, pswd, cert=None, ID=None, filename=None, title=None, version=None, typeVersion=None, Hash=None): url = host.strip("/") + "/rest/types/select" params = dict() if ID: params['id'] = ID if filename: params['filename'] = filename if title: params['title'] = title if version: params['version'] = version if typeVersion: params['typeVersion'] = typeVersion if Hash: params['hash'] = Hash r = requests.get(url, params=params, auth=(user, pswd), verify=cert) return check_response(r)
def create_mapping(self, mapping_id): url = self.host_url + self.api_version + self.federation_string + self.mappings_string + mapping_id header = self.headers.header_post header = self._check_header_auth_token(header) body = self.mapping if body["mapping"]["rules"][0]["local"][1]["group"]["id"] is None: if self.ids.group_id is None: group_name = raw_input("Insert group name: ") group_id = utils.read_group_id_by_name(self.list_groups().json(), group_name) body["mapping"]["rules"][0]["local"][1]["group"]["id"] = group_id else: body["mapping"]["rules"][0]["local"][1]["group"]["id"] = self.ids.group_id resp = requests.put(url, data=json.dumps(body), headers=header, verify=False) if utils.check_response(resp, 201): self.mapping = resp.json().get("mapping") print("HTTP Status Code: 201\nMapping created:") print(json.dumps(self.mapping, sort_keys=True, indent=4, separators=(',', ': '))) else: utils.expose_reason(resp) self.ids.mapping_id = mapping_id
def create_group(self): url = self.host_url + self.api_version + self.groups_string header = self.headers.header_post header = self._check_header_auth_token(header) body = self.group # domain_id is optional if body["group"]["domain_id"] is None: if self.ids.domain_id is None: domain_name = raw_input("Insert domain name: ") domain_id = utils.read_domain_id_by_name(self.list_groups().json(), domain_name) body["group"]["domain_id"] = domain_id else: body["group"]["domain_id"] = self.ids.domain_id resp = requests.post(url, data=json.dumps(body), headers=header, verify=False) if utils.check_response(resp, 201): self.group = resp.json().get("group") print("HTTP Status Code: 201\nGroup created:") print(json.dumps(self.group, sort_keys=True, indent=4, separators=(',', ': '))) else: utils.expose_reason(resp) self.ids.group_id = self.group["id"]
def create_protocol(self, protocol_id, idp_id): url = self.host_url + self.api_version + self.federation_string + self.idps_string + idp_id + \ self.protocols_string + protocol_id header = self.headers.header_post header = self._check_header_auth_token(header) body = self.protocol if body["protocol"]["mapping_id"] is None: if self.ids.mapping_id is None: mapping_id = raw_input("Insert mapping_id: ") body["protocol"]["mapping_id"] = mapping_id else: body["protocol"]["mapping_id"] = self.ids.mapping_id print(body) resp = requests.put(url, data=json.dumps(body), headers=header, verify=False) if utils.check_response(resp, 201): self.protocol = resp.json().get("protocol") print("HTTP Status Code: 201\nProtocol created:") print(json.dumps(self.protocol, sort_keys=True, indent=4, separators=(',', ': '))) else: utils.expose_reason(resp) self.ids.protocol_id = protocol_id
def add(filename, title, host, user, pswd, cert=None, version=None, dependencies=None): url = host.strip("/") + "/rest/types/add" with open(filename, 'r') as f: xsd_data = f.read() data = dict() data['content'] = [xsd_data] data['filename'] = [filename] data['title'] = [title] if version: data['typeVersion'] = [version] if dependencies: data['dependencies[]'] = dependencies r = requests.post(url, data=data, auth=(user, pswd), verify=cert) return check_response(r)
def select_all(host, user, pswd, cert=None, format=None): """Get all data from the MDCS server Inputs: host - string, URL of MDCS instance user - string, username of desired account on MDCS server pswd - string, password of desired account on MDCS server cert - string, path to authentication certificate format - string, format of data (can be xml or json) Output: lists where each entry is a dictionary describing entries that match the query. dictionaries each have the keys: title - title of the entry schema - ID of the schema that describes the entry content - content of the entry in either xml or json format _id - ID number of the entry """ url = host.strip("/") + "/rest/explore/select/all" params = dict() if format: params['dataformat'] = format r = requests.get(url, params=params, auth=(user, pswd), verify=cert) return check_response(r)
def select_all(host,user,pswd,cert=None,format=None): """Get all data from the MDCS server Inputs: host - string, URL of MDCS instance user - string, username of desired account on MDCS server pswd - string, password of desired account on MDCS server cert - string, path to authentication certificate format - string, format of data (can be xml or json) Output: lists where each entry is a dictionary describing entries that match the query. dictionaries each have the keys: title - title of the entry schema - ID of the schema that describes the entry content - content of the entry in either xml or json format _id - ID number of the entry """ url = host.strip("/") + "/rest/explore/select/all" params = dict() if format: params['dataformat'] = format r = requests.get(url, params=params, auth=(user, pswd), verify=cert) return check_response(r)
def versions_select_all(host, user, pswd, cert=None): url = host.strip("/") + "/rest/types/versions/select/all" r = requests.get(url, auth=(user, pswd), verify=cert) return check_response(r)
def restore(ID, host, user, pswd, cert=None): url = host.strip("/") + "/rest/types/restore" params = dict() params['id'] = ID r = requests.get(url, params=params, auth=(user, pswd), verify=cert) return check_response(r)
#while m<13: try: date_time_max = '2018-12-01T00:00:03Z' date_time_min = '2019-01-01T00:00:03Z' #date_time_max= '2018-'+str(m)+'-01T00:00:03Z' #date_time_min= '2018-'+str(m+1)+'-01T00:00:03Z' print('From ', date_time_max, ' To ', date_time_min) response = config.SF.query_all( query + 'and CreatedDate > {0} and CreatedDate < {1} )'.format( date_time_max, date_time_min)) print('Got The Records!') is_updated = utils.check_response(response) if is_updated: sys.exit() df = pd.DataFrame(response['records']) df.drop(['attributes'], axis=1, inplace=True) print('Done Processing') table_name = 'case_comment_tech_support' print('size: ', len(df)) print('case_comment_tech_support_2018 for ', str(m)) df.to_csv('case_comment_tech_support_2018_' + str(m) + '.csv') print('All done!')
def versions_select_all(host,user,pswd,cert=None): url = host.strip("/") + "/rest/types/versions/select/all" r = requests.get(url, auth=(user, pswd), verify=cert) return check_response(r)
def restore(ID,host,user,pswd,cert=None): url = host.strip("/") + "/rest/types/restore" params = dict() params['id']=ID r = requests.get(url, params=params, auth=(user, pswd), verify=cert) return check_response(r)
def upload(name, host, user, pswd, cert=None): url = host.strip("/") + "/rest/blob" files = {'blob': open(name, 'rb')} r = requests.post(url, files=files, auth=(user, pswd), verify=cert) result = check_response(r) return result['handle']
def upload(name,host,user,pswd,cert=None): url = host.strip("/") + "/rest/blob" files = {'blob':open(name, 'rb')} r = requests.post(url, files=files, auth=(user, pswd), verify=cert) result = check_response(r) return result['handle']
def send_element(element, conf, send_cache, dryrun, verbose, cache_only): """Attempts to gather and send information defined by element to server. If send_cache or dryrun is False, cache will be ignored. If cache_only is True, only cache is checked and no new data is gathered. @param element: Element class that should be fetched. @type element: L{MetaElement} sub class. @param conf: Configuration dictionary. @type conf: dict @param send_cache: Indicates whether cache should be included. @type send_cache: bool @param dryrun: Indicates whether the script is doing a dry run. @type dryrun: bool @param verbose: Indicates whether or not the script runs in verbose mode. @type verbose: bool @param cache_only: Indicates whether to send only cache for this element. @type cache_only: bool """ if dryrun and cache_only: # We're doing a dry run and we've reached the cached items return m = MetaDoc(conf.get("site_name")) element_processor = get_element_processor(element, send_cache, verbose, dryrun) if not cache_only: # If we're doing cache only, we've reached the possible_send_elements # loop and do not want or need to remove elements from it anymore. site_element = element.site_handler() site_element.populate() element_processor.add_elements(site_element.fetch()) m.reg_meta_element(element_processor) # Build URL url = "%s%s" % (conf['host'], element.url) if conf.get('trailing_slash',"").lower() == 'true'\ or conf.get('trailing_slash',"").lower() == 'yes': url = "%s/" % url # Check to see if there is any content to transfer in the MetaDoc. if m.has_content(): # Do not want to send any data if we're doing a dry run. if not dryrun: cli = metahttp.XMLClient(url, conf['key'], conf['cert']) if verbose: print "-" * 70 print "Connecting to host: %s" % url print "Using key: %s" % conf['key'] print "Using certificate: %s" % conf['cert'] print "-" * 70 print "Sent data:\n%s" % ("-" * 70) print m.get_xml(pretty=True) # Will not recieve any data on a dry run. if not dryrun: try: res = cli.send(m.get_xml()) except (urllib2.HTTPError, urllib2.URLError) as httperror: logging.error(("Unable to connect to server address \"%s\". " "Error: %s") % (url, httperror)) # Since we're unable to send the document to the server, we will # cache it so that we can send it at a later date. if not send_cache: logging.warning("Could not send data to server, " "but running with --no-cache, so data was not cached.") else: Cacher(element.xml_tag_name, m) else: if res: xml_data = res.read() if verbose: print "%s\nRecieved data:\n%s" % ("-" * 70, "-" * 70) print xml_data print "-" * 70 utils.check_response(element.xml_tag_name, m, xml_data, send_cache) else: logging.error(("Server returned an empty response when " "attempting to send \"%s\". Caching data.") % element.xml_tag_name) # Since we recieved an empty response from the server we have # not recieved any reciept for any elements and must cache # them. if not send_cache: logging.warning("No data returned from server, but " "running with --no-cache, so data was not cached.") else: Cacher(element.xml_tag_name, m) else: if verbose and not cache_only: print "No data to send for \"%s\"." % element.xml_tag_name logging.info(("No data to send for \"%s\".") % element.xml_tag_name)
def send_element(element, conf, send_cache, dryrun, verbose, cache_only): """Attempts to gather and send information defined by element to server. If send_cache is True or dryrun is False, cache will be ignored. If cache_only is True, only cache is checked and no new data is gathered. @param element: Element class that should be fetched. @type element: L{MetaElement} sub class. @param conf: Configuration dictionary. @type conf: dict @param send_cache: Indicates whether cache should be included. @type send_cache: bool @param dryrun: Indicates whether the script is doing a dry run. @type dryrun: bool @param verbose: Indicates whether or not the script runs in verbose mode. @type verbose: bool @param cache_only: Indicates whether to send only cache for this element. @type cache_only: bool """ if dryrun and cache_only: # We're doing a dry run and we've reached the cached items return m = MetaDoc(conf.get("site_name")) element_processor = get_element_processor(element, send_cache, verbose, dryrun) if not cache_only: # If we're doing cache only, we've reached the possible_send_elements # loop and do not want or need to remove elements from it anymore. site_element = element.site_handler() site_element.populate() element_processor.add_elements(site_element.fetch()) m.reg_meta_element(element_processor) # Build URL url = "%s%s" % (conf['host'], element.url) if bool_conf_value(conf.get('trailing_slash',"")): url = "%s/" % url # Check to see if there is any content to transfer in the MetaDoc. if m.has_content(): if verbose: print "-" * 70 print "Connecting to host: %s" % url print "Using key: %s" % conf['key'] print "Using certificate: %s" % conf['cert'] print "-" * 70 print "Sent data:\n%s" % ("-" * 70) print m.get_xml(pretty=True) if not dryrun: server_response = utils.send_document(url, conf['key'], conf['cert'], m) if server_response is False: if not send_cache: logging.warning("Could not send data to server, " "but running with --no-cache, so data was not cached.") else: Cacher(element.xml_tag_name, m) else: if verbose: print "%s\nRecieved data:\n%s" % ("-" * 70, "-" * 70) print server_response print "-" * 70 utils.check_response(element.xml_tag_name, m, server_response, send_cache) else: if verbose and not cache_only: print "No data to send for \"%s\"." % element.xml_tag_name logging.info(("No data to send for \"%s\".") % element.xml_tag_name)