def get_data(self, params={}):
"""
Make the request and return the deserialided JSON from the response
:param params: Dictionary mapping (string) query parameters to values
:type params: dict
:return: JSON object with the data fetched from that URL as a
JSON-format object.
:rtype: (dict or array)
"""
if self and hasattr(self, 'proxies') and self.proxies is not None:
response = requests.request('GET',
url=PVWatts.PVWATTS_QUERY_URL,
params=params,
headers={'User-Agent': ''.join(
['pypvwatts/', VERSION,
' (Python)'])},
proxies=self.proxies)
else:
response = requests.request('GET',
url=PVWatts.PVWATTS_QUERY_URL,
params=params,
headers={'User-Agent': ''.join(
['pypvwatts/', VERSION,
' (Python)'])})
if response.status_code == 403:
raise PVWattsError("Forbidden, 403")
return response.json()
def save_profile(backend, user, response, *args, **kwargs):
print "Saving profile is called"
if backend.name == 'facebook':
(profile, created) = MyUser.objects.get_or_create(user_id=user.id)
#print "User instance", user.__dict__
#print "Response instance", response
profile.name = response['name']
profile.user.email = response['email']
url = 'http://graph.facebook.com/{0}/picture'.format(response['id'])
try:
response_large = request('GET', url, params={'type': 'large'})
response_icon = request('GET', url, params={'type': 'square'})
response_large.raise_for_status()
response_icon.raise_for_status()
except HTTPError:
print "Error getting picture"
if not created:
# delete previous image
import os
from django.conf import settings
os.remove(os.path.join(settings.MEDIA_ROOT, str(profile.profile_icon.name)))
os.remove(os.path.join(settings.MEDIA_ROOT, str(profile.profile_pic.name)))
print "Deleted previous profile image and icon"
profile.profile_pic.save('{0}_social.jpg'.format(user.username),
ContentFile(response_large.content))
profile.profile_icon.save('{0}_icon.jpg'.format(user.username),
ContentFile(response_icon.content))
profile.save()
def test_create(self):
req_body = {
"user": {"name": "gabriel", "password": "******", "tenantId": 2, "email": "*****@*****.**", "enabled": True}
}
resp_body = {
"user": {
"name": "gabriel",
"enabled": True,
"tenantId": 2,
"id": 3,
"password": "******",
"email": "*****@*****.**",
}
}
resp = utils.TestResponse({"status_code": 200, "text": json.dumps(resp_body)})
kwargs = copy.copy(self.TEST_REQUEST_BASE)
kwargs["headers"] = self.TEST_POST_HEADERS
kwargs["data"] = json.dumps(req_body)
requests.request("POST", urlparse.urljoin(self.TEST_URL, "v2.0/users"), **kwargs).AndReturn((resp))
self.mox.ReplayAll()
user = self.client.users.create(
req_body["user"]["name"],
req_body["user"]["password"],
req_body["user"]["email"],
tenant_id=req_body["user"]["tenantId"],
enabled=req_body["user"]["enabled"],
)
self.assertTrue(isinstance(user, users.User))
self.assertEqual(user.id, 3)
self.assertEqual(user.name, "gabriel")
self.assertEqual(user.email, "*****@*****.**")
def perform_request(method, url, data_or_params=None, *args, **kwargs):
if data_or_params is None:
data_or_params = {}
headers = {'X-Edx-Api-Key': settings.API_KEY}
try:
with dog_stats_api.timer('comment_client.request.time'):
if method in ['post', 'put', 'patch']:
response = requests.request(method, url, data=data_or_params, headers=headers, timeout=5)
else:
response = requests.request(method, url, params=data_or_params, headers=headers, timeout=5)
except Exception as err:
log.exception("Trying to call {method} on {url} with params {params}".format(
method=method, url=url, params=data_or_params))
# Reraise with a single exception type
raise CommentClientError(str(err))
if 200 < response.status_code < 500:
raise CommentClientError(response.text)
# Heroku returns a 503 when an application is in maintenance mode
elif response.status_code == 503:
raise CommentClientMaintenanceError(response.text)
elif response.status_code == 500:
raise CommentClientUnknownError(response.text)
else:
if kwargs.get("raw", False):
return response.text
else:
return json.loads(response.text)
def test_update(self, ref=None):
ref = ref or self.new_ref()
req_ref = ref.copy()
del req_ref['id']
resp = TestResponse({
"status_code": 200,
"text": self.serialize(ref),
})
method = 'PATCH'
kwargs = copy.copy(self.TEST_REQUEST_BASE)
kwargs['headers'] = self.headers[method]
kwargs['data'] = self.serialize(req_ref)
requests.request(
method,
urlparse.urljoin(
self.TEST_URL,
'v3/%s/%s' % (self.collection_key, ref['id'])),
**kwargs).AndReturn((resp))
self.mox.ReplayAll()
returned = self.manager.update(ref['id'], **parameterize(req_ref))
self.assertTrue(isinstance(returned, self.model))
for attr in ref:
self.assertEqual(
getattr(returned, attr),
ref[attr],
'Expected different %s' % attr)
def call_api(self, method, resource, params=None, auth=None):
"""
given method, resource, params, auth - return tuple: data, error
"""
kwargs = {}
if auth:
code = base64.b64encode(self.conf.get(auth, 'user')+':'+self.conf.get(auth, 'password'))
kwargs['headers'] = {'Authorization': 'Basic '+code}
if params:
kwargs['params'] = params
try:
url = self.conf.get(self.sect, 'api_url')+':'+self.conf.get(self.sect, 'port')+'/'+resource
if kwargs:
result = requests.request(method, url, **kwargs)
else:
result = requests.request(method, url)
except Exception as e:
return None, "Unable to connect to API server: "+e
if not (result.ok and result.text):
return None, "Unable to connect to API server: "+e
rj = result.json
if not (rj and isinstance(rj, dict)):
return None, "Return data not valid JSON format"
if rj['error']:
return None, "%s (%d)"%(rj['error'], rj['status'])
return rj['data'], None
def _handle_request(self, method, url, data=None):
"""Handle actually talking out to the trakt API, logging out debug
information, raising any relevant `TraktException` Exception types,
and extracting and returning JSON data
:param method: The HTTP method we're executing on. Will be one of
post, put, delete, get
:param url: The fully qualified url to send our request to
:param data: Optional data payload to send to the API
:return: The decoded JSON response from the Trakt API
:raises TraktException: If any non-200 return code is encountered
"""
self.logger.debug('%s: %s', method, url)
HEADERS['trakt-api-key'] = CLIENT_ID
HEADERS['Authorization'] = 'Bearer {0}'.format(OAUTH_TOKEN)
self.logger.debug('headers: %s', str(HEADERS))
self.logger.debug('method, url :: %s, %s', method, url)
if method == 'get': # GETs need to pass data as params, not body
response = requests.request(method, url, params=data,
headers=HEADERS)
else:
response = requests.request(method, url, data=json.dumps(data),
headers=HEADERS)
self.logger.debug('RESPONSE [%s] (%s): %s', method, url, str(response))
if response.status_code in self.error_map:
raise self.error_map[response.status_code]()
elif response.status_code == 204: # HTTP no content
return None
json_data = json.loads(response.content.decode('UTF-8', 'ignore'))
return json_data
def best_choice(customer_id):
url = "http://metro.food-hacks.de/rpc/login"
payload = "{ \"email\": \"[email protected]\", \"pass\": \"RJbKtGCW2fyrG9hF\"}"
headers = {
'content-type': "application/json"
}
response = requests.request("POST", url, data=payload, headers=headers)
# get response json
login_response = json.loads(response.text)
# get jwt token
jwt_token = login_response['token']
sales_items_url = "http://metro.food-hacks.de/salesorder_items"
headers = {
'authorization': "Bearer {}".format(jwt_token),
'cache-control': "no-cache"
}
sales_items_url_tmp = sales_items_url + "?customer_id=eq.Client_{:04d}&select=article_name".format(customer_id)
response = requests.request("GET", sales_items_url_tmp, headers=headers)
json_array = json.loads(response.text)
string_list = list()
for json_object in json_array:
string_list.add(json_object['article_name'].encode('utf8'))
mcommon = [ite for ite, it in Counter(string_list).most_common(3)]
return mcommon
def test_get_version_local(self):
resp = utils.TestResponse({
"status_code": 300,
"text": json.dumps(self.TEST_RESPONSE_DICT),
})
kwargs = copy.copy(self.TEST_REQUEST_BASE)
kwargs['headers'] = self.TEST_REQUEST_HEADERS
requests.request('GET',
"http://localhost:35357",
**kwargs).AndReturn((resp))
self.mox.ReplayAll()
cs = client.Client()
versions = cs.discover()
self.assertIsInstance(versions, dict)
self.assertIn('message', versions)
self.assertIn('v3.0', versions)
self.assertEquals(
versions['v3.0']['url'],
self.TEST_RESPONSE_DICT['versions']['values'][0]['links'][0]
['href'])
self.assertEquals(
versions['v2.0']['url'],
self.TEST_RESPONSE_DICT['versions']['values'][1]['links'][0]
['href'])
def test_get(self):
user_id = 'usr'
tenant_id = 'tnt'
resp_body = {
"credential": {
"access": "access",
"secret": "secret",
"tenant_id": tenant_id,
"created": "12/12/12",
"enabled": True,
}
}
resp = utils.TestResponse({
"status_code": 200,
"text": json.dumps(resp_body),
})
url = urlparse.urljoin(self.TEST_URL,
'v2.0/users/%s/credentials/OS-EC2/%s' %
(user_id, 'access'))
kwargs = copy.copy(self.TEST_REQUEST_BASE)
kwargs['headers'] = self.TEST_REQUEST_HEADERS
requests.request('GET',
url,
**kwargs).AndReturn((resp))
self.mox.ReplayAll()
cred = self.client.ec2.get(user_id, 'access')
self.assertTrue(isinstance(cred, ec2.EC2))
self.assertEqual(cred.tenant_id, tenant_id)
self.assertEqual(cred.enabled, True)
self.assertEqual(cred.access, 'access')
self.assertEqual(cred.secret, 'secret')
def test_find(self, ref=None):
ref = ref or self.new_ref()
ref_list = [ref]
resp = TestResponse({
"status_code": 200,
"text": self.serialize(ref_list),
})
method = 'GET'
kwargs = copy.copy(self.TEST_REQUEST_BASE)
kwargs['headers'] = self.headers[method]
query = '?name=%s' % ref['name'] if hasattr(ref, 'name') else ''
requests.request(
method,
urlparse.urljoin(
self.TEST_URL,
'v3/%s%s' % (self.collection_key, query)),
**kwargs).AndReturn((resp))
self.mox.ReplayAll()
returned = self.manager.find(name=getattr(ref, 'name', None))
self.assertTrue(isinstance(returned, self.model))
for attr in ref:
self.assertEqual(
getattr(returned, attr),
ref[attr],
'Expected different %s' % attr)
def download_urls(project, count=0):
url = "https://your_domain_should_be_here.basecamphq.com/projects/%s/attachments?n=%s" % (project, count)
response = requests.request("GET", url, headers=headers)
while response.status_code != 200:
time.sleep(3)
response = requests.request("GET", url, headers=headers)
if response.status_code != 200:
print "ERROR:", project
return '', []
# print(response.text)
print response.status_code
tree = ET.fromstring(response.text)
files = []
attas = tree.findall('attachment')
for att in attas:
f = {}
f['size'] = int(att.find('byte-size').text)
f['id'] = att.find('id').text
f['url'] = att.find('download-url').text
files.append(f)
return response.text, files
def test_contract(self):
stub_definition_file = os.path.join(
os.environ['CONSUMER_CONTRACTS_ROOT'],
'contracts/includes/consumer2.json'
)
with open(stub_definition_file, 'r') as f:
stub_definition = json.load(f)
path = stub_definition["predicates"][0]["equals"]["path"]
method = stub_definition["predicates"][0]["equals"]["method"]
record = json.loads(stub_definition["responses"][0]["is"]["body"])
provider.DataStore.save_record(record)
contractual_response = requests.request(
method,
self.stub_host_port+path
)
actual_response = requests.request(method, self.actual_host_port+path)
self.assertEqual(
actual_response.status_code,
contractual_response.status_code
)
# The consumer shouldn't mind if the provider returns some
# extra data. Following Postel's law.
self.assertDictContainsSubset(
contractual_response.json(),
actual_response.json()
)
provider.DataStore.delete_record(record)
def test_authenticate_failure(self):
ident = self.TEST_REQUEST_BODY['auth']['identity']
ident['password']['user']['password'] = '******'
resp = utils.TestResponse({
"status_code": 401,
"text": json.dumps({
"unauthorized": {
"message": "Unauthorized",
"code": "401",
},
}),
})
kwargs = copy.copy(self.TEST_REQUEST_BASE)
kwargs['headers'] = self.TEST_REQUEST_HEADERS
kwargs['data'] = json.dumps(self.TEST_REQUEST_BODY, sort_keys=True)
requests.request('POST',
self.TEST_URL + "/auth/tokens",
**kwargs).AndReturn((resp))
self.mox.ReplayAll()
# Workaround for issue with assertRaises on python2.6
# where with assertRaises(exceptions.Unauthorized): doesn't work
# right
def client_create_wrapper():
client.Client(user_domain_name=self.TEST_DOMAIN_NAME,
username=self.TEST_USER,
password="******",
project_id=self.TEST_TENANT_ID,
auth_url=self.TEST_URL)
self.assertRaises(exceptions.Unauthorized, client_create_wrapper)
def download_files(urls, path):
url = urls['url']
size = urls['size']
fid = urls['id']
filename = fid + '_' + unquote(url).split('/')[-1]
if os.path.exists(path + '/' + filename):
if os.path.getsize(path + '/' + filename) == size:
succ_list.append(url)
return
print "Downloading... :" + filename + "\tsize: " + str(size/1024.0) + "KB"
response = requests.request("GET", url, headers=headers)
if response.status_code != 200:
time.sleep(3)
response = requests.request("GET", url, headers=headers)
if response.status_code != 200:
time.sleep(3)
response = requests.request("GET", url, headers=headers)
if response.status_code != 200:
print "ERROR", url
err_list.append(url)
return
with open(path + "/" + filename, "wb+") as f:
f.write(response.content)
succ_list.append(url)
def test_authenticate_success_token_unscoped(self):
ident = self.TEST_REQUEST_BODY['auth']['identity']
del ident['password']
ident['methods'] = ['token']
ident['token'] = {}
ident['token']['id'] = self.TEST_TOKEN
del self.TEST_REQUEST_BODY['auth']['scope']
del self.TEST_RESPONSE_DICT['token']['catalog']
self.TEST_REQUEST_HEADERS['X-Auth-Token'] = self.TEST_TOKEN
resp = utils.TestResponse({
"status_code": 200,
"text": json.dumps(self.TEST_RESPONSE_DICT),
"headers": self.TEST_RESPONSE_HEADERS,
})
kwargs = copy.copy(self.TEST_REQUEST_BASE)
kwargs['headers'] = self.TEST_REQUEST_HEADERS
kwargs['data'] = json.dumps(self.TEST_REQUEST_BODY, sort_keys=True)
requests.request('POST',
self.TEST_URL + "/auth/tokens",
**kwargs).AndReturn((resp))
self.mox.ReplayAll()
cs = client.Client(token=self.TEST_TOKEN,
auth_url=self.TEST_URL)
self.assertEqual(cs.auth_token,
self.TEST_RESPONSE_HEADERS["X-Subject-Token"])
self.assertFalse('catalog' in cs.service_catalog.catalog)
def test_authenticate_success(self):
TEST_TOKEN = "abcdef"
self.TEST_RESPONSE_HEADERS['X-Subject-Token'] = TEST_TOKEN
ident = self.TEST_REQUEST_BODY['auth']['identity']
del ident['password']['user']['domain']
del ident['password']['user']['name']
ident['password']['user']['id'] = self.TEST_USER
resp = utils.TestResponse({
"status_code": 200,
"text": json.dumps(self.TEST_RESPONSE_DICT),
"headers": self.TEST_RESPONSE_HEADERS,
})
kwargs = copy.copy(self.TEST_REQUEST_BASE)
kwargs['headers'] = self.TEST_REQUEST_HEADERS
kwargs['data'] = json.dumps(self.TEST_REQUEST_BODY, sort_keys=True)
requests.request('POST',
self.TEST_URL + "/auth/tokens",
**kwargs).AndReturn((resp))
self.mox.ReplayAll()
cs = client.Client(user_id=self.TEST_USER,
password=self.TEST_TOKEN,
project_id=self.TEST_TENANT_ID,
auth_url=self.TEST_URL)
self.assertEqual(cs.auth_token, TEST_TOKEN)
def test_authenticate_success_domain_username_password_scoped(self):
resp = utils.TestResponse({
"status_code": 200,
"text": json.dumps(self.TEST_RESPONSE_DICT),
"headers": self.TEST_RESPONSE_HEADERS,
})
kwargs = copy.copy(self.TEST_REQUEST_BASE)
kwargs['headers'] = self.TEST_REQUEST_HEADERS
kwargs['data'] = json.dumps(self.TEST_REQUEST_BODY, sort_keys=True)
requests.request('POST',
self.TEST_URL + "/auth/tokens",
**kwargs).AndReturn((resp))
self.mox.ReplayAll()
cs = client.Client(user_domain_name=self.TEST_DOMAIN_NAME,
username=self.TEST_USER,
password=self.TEST_TOKEN,
project_id=self.TEST_TENANT_ID,
auth_url=self.TEST_URL)
self.assertEqual(cs.management_url,
self.TEST_RESPONSE_DICT["token"]["catalog"][3]
['endpoints'][2]["url"])
self.assertEqual(cs.auth_token,
self.TEST_RESPONSE_HEADERS["X-Subject-Token"])
def test_authenticate_success_token_project_scoped(self):
ident = self.TEST_REQUEST_BODY['auth']['identity']
del ident['password']
ident['methods'] = ['token']
ident['token'] = {}
ident['token']['id'] = self.TEST_TOKEN
self.TEST_REQUEST_HEADERS['X-Auth-Token'] = self.TEST_TOKEN
resp = utils.TestResponse({
"status_code": 200,
"text": json.dumps(self.TEST_RESPONSE_DICT),
"headers": self.TEST_RESPONSE_HEADERS,
})
kwargs = copy.copy(self.TEST_REQUEST_BASE)
kwargs['headers'] = self.TEST_REQUEST_HEADERS
kwargs['data'] = json.dumps(self.TEST_REQUEST_BODY, sort_keys=True)
requests.request('POST',
self.TEST_URL + "/auth/tokens",
**kwargs).AndReturn((resp))
self.mox.ReplayAll()
cs = client.Client(token=self.TEST_TOKEN,
project_id=self.TEST_TENANT_ID,
auth_url=self.TEST_URL)
self.assertEqual(cs.auth_tenant_id,
self.TEST_TENANT_ID)
self.assertEqual(cs.management_url,
self.TEST_RESPONSE_DICT["token"]["catalog"][3]
['endpoints'][2]["url"])
self.assertEqual(cs.auth_token,
self.TEST_RESPONSE_HEADERS["X-Subject-Token"])
def getRealLinks(self):
realLinks = []
for links in self.pageLinks:
headers = {
'cache-control': "no-cache",
'postman-token': "8af4828e-133e-5c8f-e4bd-10ac70d6048a"
}
try:
response = requests.request("GET", links, headers=headers)
except:
response = requests.request("GET", links, headers=headers)
m = re.search('(orig=.+?"http.+?")', response.text)
try:
cleanUrl = m.group(0)[m.group(0).index("http"):]
cleanUrl = cleanUrl[:cleanUrl.rindex("\\")]
except ValueError as e:
print(response.text)
print(e)
realLinks.append(cleanUrl)
return realLinks
def catch_all(path, type):
try:
data = flask.request.environ['body_copy']
filter(type, path, data)
#PICK RANDOM ES
es = random.choice(listwrap(settings["elasticsearch"]))
## SEND REQUEST
headers = {k: v for k, v in flask.request.headers if v is not None and v != "" and v != "null"}
headers['content-type'] = 'application/json'
response = requests.request(
type,
es["host"] + ":" + str(es["port"]) + "/" + path,
data=data,
stream=True, # FOR STREAMING
headers=headers,
timeout=90
)
# ALLOW CROSS DOMAIN (BECAUSE ES IS USUALLY NOT ON SAME SERVER AS PAGE)
outbound_header = dict(response.headers)
outbound_header["access-control-allow-origin"] = "*"
# LOG REQUEST TO ES
request = flask.request
uid = int(round(time.time() * 1000.0))
slim_request = {
"remote_addr": request.remote_addr,
"method": request.method,
"path": request.path,
"request_length": len(data),
"response_length": int(outbound_header["content-length"]) if "content-length" in outbound_header else None
}
try:
requests.request(
type,
es["host"] + ":" + str(es["port"]) + "/debug/esfrontline/"+str(uid),
data=json.dumps(slim_request),
timeout=5
)
except Exception, e:
pass
logger.debug("path: {path}, request bytes={request_content_length}, response bytes={response_content_length}".format(
path=path,
# request_headers=dict(response.headers),
request_content_length=len(data),
# response_headers=outbound_header,
response_content_length=int(outbound_header["content-length"]) if "content-length" in outbound_header else None
))
## FORWARD RESPONSE
return flask.wrappers.Response(
stream(response.raw),
direct_passthrough=True, #FOR STREAMING
status=response.status_code,
headers=outbound_header
)
def prod_api(token):
non_gzipped_headers = {
'Authorization': token,
'Accept': 'application/json, */*'
}
gzipped_headers = {
'Authorization': token,
'Accept': 'application/json, */*',
'Accept-Encoding': 'gzip'
}
urls = [
"/api/gis/coordinates/",
"/api/gis/county_boundaries/",
"/api/gis/ward_boundaries/",
"/api/gis/constituency_boundaries/",
"/api/common/filtering_summaries/"
"/api/facilities/facilities/",
"/api/facilities/facilities_list/",
"/api/facilities/facilities_list/?format=excel"
]
for i in urls:
# warmup non-gzip encoded content
requests.request(
"GET", url=_get_url(i), headers=non_gzipped_headers
)
# warmup gzip encoded content
requests.request("GET", url=_get_url(i), headers=gzipped_headers)
def load(method, url, data):
if method in ['GET', 'DELETE']:
print 'FETCH %s %s' % (url, data)
response = requests.request(method, url, params=data, allow_redirects=True)
if method in ['POST', 'PUT']:
files = {}
for key in data:
if hasattr(data[key], 'read'):
files[key] = data[key]
for key in files:
data.pop(key)
print 'FETCH %s %s' % (url, data)
response = requests.request(method, url, data=data, files=files)
result = self._parse(response.content)
try:
next_url = result['paging']['next']
except (KeyError, TypeError):
next_url = None
return result, next_url
def load(method, url, data):
if method in ['GET', 'DELETE']:
try:
response = requests.request(method, url, params=data, allow_redirects=True)
except requests.RequestException as exception:
raise self.HTTPError(exception.message)
if method in ['POST', 'PUT']:
files = {}
for key in data:
if hasattr(data[key], 'read'):
files[key] = data[key]
for key in files:
data.pop(key)
try:
response = requests.request(method, url, data=data, files=files)
except requests.RequestException as exception:
raise self.HTTPError(exception.message)
result = self._parse(response.content)
try:
next_url = result['paging']['next']
except (KeyError, TypeError):
next_url = None
return result, next_url
def request(method, url, **kwargs):
try:
_L.debug("Requesting %s with args %s", url, kwargs.get('params') or kwargs.get('data'))
return requests.request(method, url, timeout=_http_timeout, **kwargs)
except requests.exceptions.SSLError as e:
_L.warning("Retrying %s without SSL verification", url)
return requests.request(method, url, timeout=_http_timeout, verify=False, **kwargs)
def test_authenticate_success_token_scoped(self):
del self.TEST_REQUEST_BODY['auth']['passwordCredentials']
self.TEST_REQUEST_BODY['auth']['token'] = {'id': self.TEST_TOKEN}
self.TEST_REQUEST_HEADERS['X-Auth-Token'] = self.TEST_TOKEN
resp = utils.TestResponse({
"status_code": 200,
"text": json.dumps(self.TEST_RESPONSE_DICT),
})
kwargs = copy.copy(self.TEST_REQUEST_BASE)
kwargs['headers'] = self.TEST_REQUEST_HEADERS
kwargs['data'] = json.dumps(self.TEST_REQUEST_BODY)
requests.request('POST',
self.TEST_URL + "/tokens",
**kwargs).AndReturn((resp))
self.mox.ReplayAll()
cs = client.Client(token=self.TEST_TOKEN,
tenant_id=self.TEST_TENANT_ID,
auth_url=self.TEST_URL)
self.assertEqual(cs.management_url,
self.TEST_RESPONSE_DICT["access"]["serviceCatalog"][3]
['endpoints'][0]["adminURL"])
self.assertEqual(cs.auth_token,
self.TEST_RESPONSE_DICT["access"]["token"]["id"])
def test_authenticate_failure(self):
_auth = 'auth'
_cred = 'passwordCredentials'
_pass = '******'
self.TEST_REQUEST_BODY[_auth][_cred][_pass] = 'bad_key'
resp = utils.TestResponse({
"status_code": 401,
"text": json.dumps({
"unauthorized": {
"message": "Unauthorized",
"code": "401",
},
}),
})
kwargs = copy.copy(self.TEST_REQUEST_BASE)
kwargs['headers'] = self.TEST_REQUEST_HEADERS
kwargs['data'] = json.dumps(self.TEST_REQUEST_BODY)
requests.request('POST',
self.TEST_URL + "/tokens",
**kwargs).AndReturn((resp))
self.mox.ReplayAll()
# Workaround for issue with assertRaises on python2.6
# where with assertRaises(exceptions.Unauthorized): doesn't work
# right
def client_create_wrapper():
client.Client(username=self.TEST_USER,
password="******",
tenant_id=self.TEST_TENANT_ID,
auth_url=self.TEST_URL)
self.assertRaises(exceptions.Unauthorized, client_create_wrapper)
def test_update_own_password(self):
req_body = {
'user': {
'password': '******', 'original_password': '******'
}
}
resp_body = {
'access': {}
}
resp = utils.TestResponse({
"status_code": 200,
"text": json.dumps(resp_body)
})
kwargs = copy.copy(self.TEST_REQUEST_BASE)
kwargs['headers'] = self.TEST_POST_HEADERS
kwargs['data'] = json.dumps(req_body)
requests.request(
'PATCH',
urlparse.urljoin(self.TEST_URL, 'v2.0/OS-KSCRUD/users/123'),
**kwargs).AndReturn((resp))
self.mox.ReplayAll()
self.client.user_id = '123'
self.client.users.update_own_password('DCBA', 'ABCD')
def _request(self, method, url_path, data):
url = urljoin(self.server, url_path)
headers = {
'Content-Type': 'application/json'
}
if self.token:
headers['Authorization'] = 'Bearer {}'.format(self.token)
if method == 'GET':
response = requests.request(
method, url, params=data, headers=headers,
verify=False)
else:
response = requests.request(
method, url, data=data, headers=headers,
verify=False)
result = json.loads(response.content.decode('utf-8'))
if response.status_code not in [200, 201]:
logger.debug(' {status}: "{message}"'.format(**result))
else:
logger.debug(' Success')
return result
def api_call(path, method='GET', query=None, data=None, token=None, flag_full_path=False):
# Translates all the HTTP calls to interface with the API
data = json.dumps(data) if isinstance(data, dict) or isinstance(data,list) else None
base_url = 'https://cgc-api.sbgenomics.com/v2/'
if token == None:
if 'AUTH_TOKEN' in os.environ.keys():
token = os.environ['AUTH_TOKEN']
else:
print("""
You have failed to set your AUTH_TOKEN, we can not continue.
Please go to set_AUTH_TOKEN.ipynb and set it properly.
""")
return False
headers = {
'X-SBG-Auth-Token': token,
'Accept': 'application/json',
'Content-type': 'application/json',
}
if flag_full_path:
response = request(method, path, params=query, data=data, headers=headers)
else:
response = request(method, base_url + path, params=query, data=data, headers=headers)
response_dict = json.loads(response.content) if response.content else {}
if response.status_code / 100 != 2:
print(response_dict['message'])
print('Error Code: %i.' % (response_dict['code']))
print(response_dict['more_info'])
raise Exception('Server responded with status code %s.' % response.status_code)
return response_dict
def lambda_handler(event, context):
S_and_P = [
'MMM', 'ABT', 'ABBV', 'ABMD', 'ACN', 'ATVI', 'ADBE', 'AMD', 'AAP',
'AES', 'AMG', 'AFL', 'A', 'APD', 'AKAM', 'ALK', 'ALB', 'ARE', 'ALXN',
'ALGN', 'ALLE', 'AGN', 'ADS', 'LNT', 'ALL', 'GOOGL', 'GOOG', 'MO',
'AMZN', 'AMCR', 'AEE', 'AAL', 'AEP', 'AXP', 'AIG', 'AMT', 'AWK', 'AMP',
'ABC', 'AME', 'AMGN', 'APH', 'ADI', 'ANSS', 'ANTM', 'AON', 'AOS',
'APA', 'AIV', 'AAPL', 'AMAT', 'APTV', 'ADM', 'ARNC', 'ANET', 'AJG',
'AIZ', 'ATO', 'T', 'ADSK', 'ADP', 'AZO', 'AVB', 'AVY', 'BLL', 'BAC',
'BK', 'BAX', 'BBT', 'BDX', 'BRK.B', 'BBY', 'BHGE', 'BIIB', 'BLK',
'HRB', 'BA', 'BKNG', 'BWA', 'BXP', 'BSX', 'BMY', 'AVGO', 'BR', 'BF.B',
'CHRW', 'COG', 'CDNS', 'CPB', 'COF', 'CPRI', 'CAH', 'KMX', 'CCL',
'CAT', 'CBOE', 'CBRE', 'CBS', 'CDW', 'CE', 'CELG', 'CNC', 'CNP', 'CTL',
'CERN', 'CF', 'SCHW', 'CHTR', 'CVX', 'CMG', 'CB', 'CHD', 'CI', 'XEC',
'CINF', 'CTAS', 'CSCO', 'C', 'CFG', 'CTXS', 'CLX', 'CME', 'CMS', 'KO',
'CTSH', 'CL', 'CMCSA', 'CMA', 'CAG', 'CXO', 'COP', 'ED', 'STZ', 'COO',
'CPRT', 'GLW', 'CTVA', 'COST', 'COTY', 'CCI', 'CSX', 'CMI', 'CVS',
'DHI', 'DHR', 'DRI', 'DVA', 'DE', 'DAL', 'XRAY', 'DVN', 'FANG', 'DLR',
'DFS', 'DISCA', 'DISCK', 'DISH', 'DG', 'DLTR', 'D', 'DOV', 'DOW',
'DTE', 'DUK', 'DRE', 'DD', 'DXC', 'ETFC', 'EMN', 'ETN', 'EBAY', 'ECL',
'EIX', 'EW', 'EA', 'EMR', 'ETR', 'EOG', 'EFX', 'EQIX', 'EQR', 'ESS',
'EL', 'EVRG', 'ES', 'RE', 'EXC', 'EXPE', 'EXPD', 'EXR', 'XOM', 'FFIV',
'FB', 'FAST', 'FRT', 'FDX', 'FIS', 'FITB', 'FE', 'FRC', 'FISV', 'FLT',
'FLIR', 'FLS', 'FMC', 'F', 'FTNT', 'FTV', 'FBHS', 'FOXA', 'FOX', 'BEN',
'FCX', 'GPS', 'GRMN', 'IT', 'GD', 'GE', 'GIS', 'GM', 'GPC', 'GILD',
'GL', 'GPN', 'GS', 'GWW', 'HAL', 'HBI', 'HOG', 'HIG', 'HAS', 'HCA',
'HP', 'HSIC', 'HSY', 'HES', 'HPE', 'HLT', 'HFC', 'HOLX', 'HD', 'HON',
'HRL', 'HST', 'HPQ', 'HUM', 'HBAN', 'HII', 'IEX', 'IDXX', 'INFO',
'ITW', 'ILMN', 'IR', 'INTC', 'ICE', 'IBM', 'INCY', 'IP', 'IPG', 'IFF',
'INTU', 'ISRG', 'IVZ', 'IPGP', 'IQV', 'IRM', 'JKHY', 'JEC', 'JBHT',
'SJM', 'JNJ', 'JCI', 'JPM', 'JNPR', 'KSU', 'K', 'KEY', 'KEYS', 'KMB',
'KIM', 'KMI', 'KLAC', 'KSS', 'KHC', 'KR', 'LB', 'LHX', 'LH', 'LRCX',
'LW', 'LEG', 'LDOS', 'LEN', 'LLY', 'LNC', 'LIN', 'LKQ', 'LMT', 'L',
'LOW', 'LYB', 'MTB', 'MAC', 'M', 'MRO', 'MPC', 'MKTX', 'MAR', 'MMC',
'MLM', 'MAS', 'MA', 'MKC', 'MXIM', 'MCD', 'MCK', 'MDT', 'MRK', 'MET',
'MTD', 'MGM', 'MCHP', 'MU', 'MSFT', 'MAA', 'MHK', 'TAP', 'MDLZ',
'MNST', 'MCO', 'MS', 'MOS', 'MSI', 'MSCI', 'MYL', 'NDAQ', 'NOV',
'NKTR', 'NTAP', 'NFLX', 'NWL', 'NEM', 'NWSA', 'NWS', 'NEE', 'NLSN',
'NKE', 'NI', 'NBL', 'JWN', 'NSC', 'NTRS', 'NOC', 'NCLH', 'NRG', 'NUE',
'NVDA', 'NVR', 'ORLY', 'OXY', 'OMC', 'OKE', 'ORCL', 'PCAR', 'PKG',
'PH', 'PAYX', 'PYPL', 'PNR', 'PBCT', 'PEP', 'PKI', 'PRGO', 'PFE', 'PM',
'PSX', 'PNW', 'PXD', 'PNC', 'PPG', 'PPL', 'PFG', 'PG', 'PGR', 'PLD',
'PRU', 'PEG', 'PSA', 'PHM', 'PVH', 'QRVO', 'PWR', 'QCOM', 'DGX', 'RL',
'RJF', 'RTN', 'O', 'REG', 'REGN', 'RF', 'RSG', 'RMD', 'RHI', 'ROK',
'ROL', 'ROP', 'ROST', 'RCL', 'CRM', 'SBAC', 'SLB', 'STX', 'SEE', 'SRE',
'SHW', 'SPG', 'SWKS', 'SLG', 'SNA', 'SO', 'LUV', 'SPGI', 'SWK', 'SBUX',
'STT', 'SYK', 'STI', 'SIVB', 'SYF', 'SNPS', 'SYY', 'TMUS', 'TROW',
'TTWO', 'TPR', 'TGT', 'TEL', 'FTI', 'TFX', 'TXN', 'TXT', 'TMO', 'TIF',
'TWTR', 'TJX', 'TSCO', 'TDG', 'TRV', 'TRIP', 'TSN', 'UDR', 'ULTA',
'USB', 'UAA', 'UA', 'UNP', 'UAL', 'UNH', 'UPS', 'URI', 'UTX', 'UHS',
'UNM', 'VFC', 'VLO', 'VAR', 'VTR', 'VRSN', 'VRSK', 'VZ', 'VRTX',
'VIAB', 'V', 'VNO', 'VMC', 'WAB', 'WMT', 'WBA', 'DIS', 'WM', 'WAT',
'WEC', 'WCG', 'WFC', 'WELL', 'WDC', 'WU', 'WRK', 'WY', 'WHR', 'WMB',
'WLTW', 'WYNN', 'XEL', 'XRX', 'XLNX', 'XYL', 'YUM', 'ZBH', 'ZION',
'ZTS'
]
baseURL1 = "https://financialmodelingprep.com/api/v3/financials/income-statement/"
baseURL2 = "https://financialmodelingprep.com/api/v3/financials/balance-sheet-statement/"
baseURL3 = "https://financialmodelingprep.com/api/v3/financials/cash-flow-statement/"
with open("/tmp/" + date.today().strftime("%Y-%m-%d") + ".json",
mode='w',
encoding='utf-8') as feedsjson:
for i in S_and_P:
dataHolder = ""
url = baseURL1 + i + "?period=quarter"
response = requests.request("GET", url)
dataHolder = dataHolder + response.text
feedsjson.write(dataHolder)
feedsjson.write("\n")
dataHolder = ""
url = baseURL2 + i + "?period=quarter"
response = requests.request("GET", url)
dataHolder = dataHolder + response.text
feedsjson.write(dataHolder)
feedsjson.write("\n")
dataHolder = ""
url = baseURL3 + i + "?period=quarter"
response = requests.request("GET", url)
dataHolder = dataHolder + response.text
feedsjson.write(dataHolder)
feedsjson.write("\n")
s3 = boto3.client('s3')
filename = date.today().strftime("%Y-%m-%d") + ".json"
bucket_name = 'bdai-data'
s3.upload_file("/tmp/" + filename, bucket_name,
"monthly-financial-data/" + filename)
return {'statusCode': 200, 'body': filename}
def getOrg():
send_url = "http://127.0.0.1:5002/api/v1/vm/org"
# send_url = "http://172.16.39.154:5555/api/v1/vcloud/get_org"
return_data = requests.request("GET", send_url)
return_data = json.loads(return_data.text)
return json_response({"result": return_data}, 200)
def getAggregate():
# return_result = aggregateValue()
send_url = "http://127.0.0.1:5002/api/v1/volume/aggregate"
return_result = requests.request("GET", send_url)
return_result = json.loads(return_result.text)
return json_response({"result": return_result}, 200)
def checkVolume():
# return_result = volumeName()
send_url = "http://127.0.0.1:5002/api/v1/volume"
return_result = requests.request("GET", send_url)
return_result = json.loads(return_result.text)
return json_response({"result": return_result}, 200)
def provisioning(body_data):
send_url = "http://127.0.0.1:5002/api/v1/vm"
# body_data = request.get_json()
headers = {"platform": "flexpod"}
# data = {
# # data that i want
# "name": "L_vm",
# "storage": "datastore-12",
# "storageType": "thin",
# "templateId": "04d9ed8b-03d1-4e31-934b-ca8962de63ff",
# "templateName": "Fortigate",
# # "template_name": "Template_Windows2012_prd_api",
# "networkId": "Don't know",
# "networkName": "VXLAN-DVUplinks-25 or dvportgroup-26",
# "username": "******",
# "customerName": "L_cust_test",
# "orgName": "L_org_test",
# "vdcName": "L_vdc_test",
# "vAppName": "L_vapp_test",
# "vAppUser": "******",
# "vAppPassword": "******",
# "vAppPasswordConfirm": "L_vapp_pass",
# "zone": "POC"
# }
# body_data = data
# checking_key, missing_key = checkParameter(body_data.keys(), ["name", "storage", "storageType", "templateId",
# "templateName", "networkId", "networkName",
# "customerName])
# if checking_key:
# pass
# else:
# return json_response({"message": "Missing parameters: " + ",".join(missing_key)}, 400)
# data = {
# # data that i want
# "name": "L_SampleVM",
# "serviceOs": "WINDOWS_8_64",
# "datastore": "datastore-11",
# "folder": "group-v3",
# "resourcePool": "resgroup-21",
# "memorySize": "4 GB",
# "cpuCount": "2 Core",
# "diskCapacity": "20 GB",
# "networkType": "E1000",
# "backingType": "STANDARD_PORTGROUP",
# "network": "network-13",
# "customerName": "L_cust_test",
# "orgName": "L_org_test",
# "vdcName": "L_vdc_test",
# "vAppName": "L_vapp_test",
# "vAppUser": "******",
# "vAppPassword": "******",
# "vAppPasswordConfirm": "L_vapp_pass",
# "zone": "POC"
# }
# body_data = data
# checking_key, missing_key = checkParameter(body_data.keys(), ["name", "serviceOs", "datastore", "folder",
# "resourcePool", "memorySize", "cpuCount",
# "diskCapacity", "networkType", "backingType",
# "network", "customerName"])
# if checking_key:
# pass
# else:
# return json_response({"message": "Missing parameters: " + ",".join(missing_key)}, 400)
# body_data = request.get_json()
# vm_name = body_data["name"]
vapp_password = body_data["vAppPassword"]
vapp_password_confirm = body_data["vAppPasswordConfirm"]
if vapp_password != vapp_password_confirm:
return "Vapp password doesn't match"
# return_result, return_data = sendData(body_data)
return_data = requests.request("POST",
send_url,
json=body_data,
headers=headers)
return_data = json.loads(return_data.text)
status_code = return_data["statusCode"]
del return_data["statusCode"]
# return json_response({"message": "Your vm name {} has been deploy".format(vm_name)}, 201)
return json_response({"result": return_data}, status_code)
import requests
url = "http://0.0.0.0:5553/invocations"
payload = "{\n \"columns\": [\n \"alcohol\",\n \"chlorides\",\n \"citric acid\",\n \"density\",\n \"fixed acidity\",\n \"free sulfur dioxide\",\n \"pH\",\n \"residual sugar\",\n \"sulphates\",\n \"total sulfur dioxide\",\n \"volatile acidity\"\n ],\n \"data\": [\n [\n 12.8,\n 0.029,\n 0.48,\n 0.98,\n 6.2,\n 29,\n 3.33,\n 1.4,\n 0.39,\n 75,\n 0.66\n ]\n ]\n}"
headers = {
'Content-Type': 'application/json; format=pandas-split'
}
response = requests.request("POST", url, headers=headers, data = payload)
print(response.text)
def call(self, method, path, params=None, **kwargs):
print(self.appAPIkey)
resp = requests.request(method, f'{self.baseURL}/{path}', params=params, headers={"Content-Type": "application/json"}, auth=HTTPBasicAuth(self.appAPIkey, ''), **kwargs)
if resp.status_code == 200 or resp.status_code == 201:
return resp.json()
raise Exception(resp.status_code, resp.json())
date_converted5 = str(date_converted4) # Converting date to string
# Date_converted5 example output: "2019-09-16 00:00:00"
date_converted6 = date_converted5[:-9] # Deleting last 9 char.
payload_conv["Event"]["date"] = date_converted6
payload_conv_json = json.dumps(payload_conv)
#print(payload_conv_json)
headers = {
'authorization': "lCRV34i1zPRryyrlAQSh0dWKZsI9VlTU2TPqquYN",
'accept': "application/json",
'content-type': "application/json",
'cache-control': "no-cache",
}
response = requests.request("POST",
url,
data=payload_conv_json,
headers=headers,
verify=False)
geteventid = json.loads(response.text) # getting event id
geteventid2 = geteventid["Event"]["id"]
geteventname = geteventid["Event"]["info"]
#print(geteventid2)
##### Splitting IP's from comma seperated values to match JSON format
ip = (pub_name_alim["ip"])
ip_comma = ip.split(",")
#print(ip_comma)
vv = list(ip_comma)
#print(vv[1)
def _do_request(self,
uri,
params={},
headers={},
type='POST',
preuri="https://www.googleapis.com"):
""" Execute the request to Google API. Return a tuple ('HTTP_CODE', 'HTTP_RESPONSE')
:param uri : the url to contact
:param params : dict or already encoded parameters for the request to make
:param headers : headers of request
:param type : the method to use to make the request
:param preuri : pre url to prepend to param uri.
"""
_logger.debug("Uri: %s - Type : %s - Headers: %s - Params : %s !",
(uri, type, headers, params))
ask_time = fields.Datetime.now()
try:
if type.upper() in ('GET', 'DELETE'):
res = requests.request(type.lower(),
preuri + uri,
params=params,
timeout=TIMEOUT)
elif type.upper() in ('POST', 'PATCH', 'PUT'):
res = requests.request(type.lower(),
preuri + uri,
data=params,
headers=headers,
timeout=TIMEOUT)
else:
raise Exception(
_('Method not supported [%s] not in [GET, POST, PUT, PATCH or DELETE]!'
) % (type))
res.raise_for_status()
status = res.status_code
if int(status) in (204, 404): # Page not found, no response
response = False
else:
response = res.json()
try:
ask_time = datetime.strptime(res.headers.get('date'),
"%a, %d %b %Y %H:%M:%S %Z")
except:
pass
except requests.HTTPError as error:
# https://developers.google.com/calendar/v3/errors
if error.response.status_code in (204, 404):
status = error.response.status_code
response = ""
else:
req = json.loads(error.request.body)
res = error.response.json()
_logger.exception(
"Error while requesting Google Services\nRequest:\n%s\nResponse:\n%s",
pformat(req), pformat(res))
if error.response.status_code in (400, 401, 410):
raise UserError(
_("Error while requesting Google Services: %s" %
res['error']['message']))
raise self.env['res.config.settings'].get_config_warning(
_("Something went wrong with your request to google"))
return (status, response, ask_time)
def format_results(client, uuid):
# Scan Lists sometimes returns empty
num_of_attempts = 0
relationships = []
response = urlscan_submit_request(client, uuid)
scan_lists = response.get('lists')
while scan_lists is None:
try:
num_of_attempts += 1
demisto.debug('Attempting to get scan lists {} times'.format(num_of_attempts))
response = urlscan_submit_request(client, uuid)
scan_lists = response.get('lists')
except Exception:
if num_of_attempts == 5:
break
demisto.debug('Could not get scan lists, sleeping for 5 minutes before trying again')
time.sleep(5)
scan_data = response.get('data', {})
scan_lists = response.get('lists', {})
scan_tasks = response.get('task', {})
scan_page = response.get('page', {})
scan_stats = response.get('stats', {})
scan_meta = response.get('meta', {})
url_query = scan_tasks.get('url', {})
scan_verdicts = response.get('verdicts', {})
ec = makehash()
dbot_score = makehash()
human_readable = makehash()
cont = makehash()
file_context = makehash()
url_cont = makehash()
feed_related_indicators = []
LIMIT = int(demisto.args().get('limit', 20))
if 'certificates' in scan_lists:
cert_md = []
cert_ec = []
certs = scan_lists['certificates']
for x in certs[:LIMIT]:
info, ec_info = cert_format(x)
cert_md.append(info)
cert_ec.append(ec_info)
CERT_HEADERS = ['Subject Name', 'Issuer', 'Validity']
cont['Certificates'] = cert_ec
url_cont['Data'] = url_query
if 'urls' in scan_lists:
url_cont['Data'] = demisto.args().get('url')
cont['URL'] = demisto.args().get('url')
if isinstance(scan_lists.get('urls'), list):
for url in scan_lists['urls']:
feed_related_indicators.append({'value': url, 'type': 'URL'})
# effective url of the submitted url
human_readable['Effective URL'] = scan_page.get('url')
cont['EffectiveURL'] = scan_page.get('url')
if 'uuid' in scan_tasks:
ec['URLScan']['UUID'] = scan_tasks['uuid']
if 'ips' in scan_lists:
ip_asn_MD = []
ip_ec_info = makehash()
ip_list = scan_lists['ips']
asn_list = scan_lists['asns']
ip_asn_dict = dict(zip(ip_list, asn_list))
i = 1
for k in ip_asn_dict:
if i - 1 == LIMIT:
break
v = ip_asn_dict[k]
ip_info = {
'Count': i,
'IP': k,
'ASN': v
}
ip_ec_info[i]['IP'] = k
ip_ec_info[i]['ASN'] = v
ip_asn_MD.append(ip_info)
i = i + 1
cont['RelatedIPs'] = ip_ec_info
if isinstance(scan_lists.get('ips'), list):
for ip in scan_lists.get('ips'):
feed_related_indicators.append({'value': ip, 'type': 'IP'})
IP_HEADERS = ['Count', 'IP', 'ASN']
# add redirected URLs
if 'requests' in scan_data:
redirected_urls = []
for o in scan_data['requests']:
if 'redirectResponse' in o['request']:
if 'url' in o['request']['redirectResponse']:
url = o['request']['redirectResponse']['url']
redirected_urls.append(url)
cont['RedirectedURLs'] = redirected_urls
if 'countries' in scan_lists:
countries = scan_lists['countries']
human_readable['Associated Countries'] = countries
cont['Country'] = countries
if None not in scan_lists.get('hashes', []):
hashes = scan_lists.get('hashes', [])
cont['RelatedHash'] = hashes
human_readable['Related Hashes'] = hashes
for hashe in hashes:
feed_related_indicators.append({'value': hashe, 'type': 'File'})
if 'domains' in scan_lists:
subdomains = scan_lists.get('domains', [])
cont['Subdomains'] = subdomains
human_readable['Subdomains'] = subdomains
for domain in subdomains:
feed_related_indicators.append({'value': domain, 'type': 'Domain'})
if 'linkDomains' in scan_lists:
link_domains = scan_lists.get('domains', [])
for domain in link_domains:
feed_related_indicators.append({'value': domain, 'type': 'Domain'})
if 'asn' in scan_page:
cont['ASN'] = scan_page['asn']
url_cont['ASN'] = scan_page.get('asn')
if 'asnname' in scan_page:
url_cont['ASOwner'] = scan_page['asnname']
if 'country' in scan_page:
url_cont['Geo']['Country'] = scan_page['country']
if 'domain' in scan_page:
feed_related_indicators.append({'value': scan_page['domain'], 'type': 'Domain'})
if 'ip' in scan_page:
feed_related_indicators.append({'value': scan_page['ip'], 'type': 'IP'})
if 'url' in scan_page:
feed_related_indicators.append({'value': scan_page['url'], 'type': 'URL'})
if 'overall' in scan_verdicts:
human_readable['Malicious URLs Found'] = scan_stats['malicious']
if scan_verdicts['overall'].get('malicious'):
human_readable['Malicious'] = 'Malicious'
url_cont['Data'] = demisto.args().get('url')
cont['Data'] = demisto.args().get('url')
dbot_score['Indicator'] = demisto.args().get('url')
url_cont['Malicious']['Vendor'] = 'urlscan.io'
cont['Malicious']['Vendor'] = 'urlscan.io'
dbot_score['Vendor'] = 'urlscan.io'
url_cont['Malicious']['Description'] = 'Match found in Urlscan.io database'
cont['Malicious']['Description'] = 'Match found in Urlscan.io database'
dbot_score['Score'] = 3
dbot_score['Type'] = 'url'
else:
dbot_score['Vendor'] = 'urlscan.io'
dbot_score['Indicator'] = demisto.args().get('url')
dbot_score['Score'] = 0
dbot_score['Type'] = 'url'
human_readable['Malicious'] = 'Benign'
dbot_score['Reliability'] = client.reliability
if 'urlscan' in scan_verdicts and 'tags' in scan_verdicts['urlscan']:
url_cont['Tags'] = scan_verdicts['urlscan']['tags']
processors_data = scan_meta['processors']
if 'download' in processors_data and len(scan_meta['processors']['download']['data']) > 0:
meta_data = processors_data['download']['data'][0]
sha256 = meta_data['sha256']
filename = meta_data['filename']
filesize = meta_data['filesize']
filetype = meta_data['mimeType']
human_readable['File']['Hash'] = sha256
cont['File']['Hash'] = sha256
file_context['SHA256'] = sha256
human_readable['File']['Name'] = filename
cont['File']['FileName'] = filename
file_context['Name'] = filename
human_readable['File']['Size'] = filesize
cont['File']['FileSize'] = filesize
file_context['Size'] = filesize
human_readable['File']['Type'] = filetype
cont['File']['FileType'] = filetype
file_context['Type'] = filetype
file_context['Hostname'] = demisto.args().get('url')
if feed_related_indicators:
related_indicators = []
for related_indicator in feed_related_indicators:
related_indicators.append(Common.FeedRelatedIndicators(value=related_indicator['value'],
indicator_type=related_indicator['type']))
url_cont['FeedRelatedIndicators'] = related_indicators
if demisto.params().get('create_relationships') is True:
relationships = create_list_relationships({'page': scan_page}, url_query,
client.reliability)
outputs = {
'URLScan(val.URL && val.URL == obj.URL)': cont,
outputPaths['file']: file_context
}
if 'screenshotURL' in scan_tasks:
human_readable['Screenshot'] = scan_tasks['screenshotURL']
screen_path = scan_tasks['screenshotURL']
response_img = requests.request("GET", screen_path, verify=client.use_ssl)
stored_img = fileResult('screenshot.png', response_img.content)
dbot_score = Common.DBotScore(indicator=dbot_score.get('Indicator'), indicator_type=dbot_score.get('Type'),
integration_name=BRAND, score=dbot_score.get('Score'),
reliability=dbot_score.get('Reliability'))
url = Common.URL(url=url_cont.get('Data'), dbot_score=dbot_score, relationships=relationships,
feed_related_indicators=url_cont.get('FeedRelatedIndicators'))
command_result = CommandResults(
readable_output=tableToMarkdown('{} - Scan Results'.format(url_query), human_readable),
outputs=outputs,
indicator=url,
raw_response=response,
relationships=relationships
)
demisto.results(command_result.to_context())
if len(cert_md) > 0:
demisto.results({
'Type': entryTypes['note'],
'ContentsFormat': formats['markdown'],
'Contents': tableToMarkdown('Certificates', cert_md, CERT_HEADERS),
'HumanReadable': tableToMarkdown('Certificates', cert_md, CERT_HEADERS)
})
if 'ips' in scan_lists:
if isinstance(scan_lists.get('ips'), list):
feed_related_indicators += scan_lists.get('ips')
demisto.results({
'Type': entryTypes['note'],
'ContentsFormat': formats['markdown'],
'Contents': tableToMarkdown('Related IPs and ASNs', ip_asn_MD, IP_HEADERS),
'HumanReadable': tableToMarkdown('Related IPs and ASNs', ip_asn_MD, IP_HEADERS)
})
if 'screenshotURL' in scan_tasks:
demisto.results({
'Type': entryTypes['image'],
'ContentsFormat': formats['text'],
'File': stored_img['File'],
'FileID': stored_img['FileID'],
'Contents': ''
})
def make_request(self, extra_data, method="GET", **kwargs):
"""Make request to Amazon MWS API with these parameters
"""
# Remove all keys with an empty value because
# Amazon's MWS does not allow such a thing.
extra_data = remove_empty(extra_data)
# -G: storing timestamp for use later
now = self.get_timestamp()
params = {
'AWSAccessKeyId': self.access_key,
self.ACCOUNT_TYPE: self.account_id,
'SignatureVersion': '2',
'Timestamp': now, # -G: calling that stored timestamp
'Version': self.version,
'SignatureMethod': 'HmacSHA256',
}
if self.auth_token:
params['MWSAuthToken'] = self.auth_token
params.update(extra_data)
# -G: urllib in Py3 has uses the module '.parse' for this functionality.
# Injecting this module into the urllib calls on both lines
# -G: removing the encoding from the parse.quote call (was `.encode('utf-8')`)
request_description = '&'.join(
['%s=%s' % (k, urllib.parse.quote(params[k], safe='-_.~')) for k in sorted(params)])
signature = self.calc_signature(method, request_description)
url = '%s%s?%s&Signature=%s' % (self.domain, self.uri, request_description, urllib.parse.quote(signature))
headers = {'User-Agent': 'python-amazon-mws/0.0.1 (Language=Python)'}
headers.update(kwargs.get('extra_headers', {}))
try:
# Some might wonder as to why i don't pass the params dict as the params argument to request.
# My answer is, here i have to get the url parsed string of params in order to sign it, so
# if i pass the params dict as params to request, request will repeat that step because it will need
# to convert the dict to a url parsed string, so why do it twice if i can just pass the full url :).
response = request(method, url, data=kwargs.get('body', ''), headers=headers)
response.raise_for_status()
# When retrieving data from the response object,
# be aware that response.content returns the content in bytes while response.text calls
# response.content and converts it to unicode.
data = response.content
#data = response.text
# I do not check the headers to decide which content structure to server simply because sometimes
# Amazon's MWS API returns XML error responses with "text/plain" as the Content-Type.
try:
parsed_response = DictWrapper(data, extra_data.get("Action") + "Result")
except XMLError:
parsed_response = DataWrapper(data, response.headers)
# Alex:
except UnicodeDecodeError:
parsed_response = data
return parsed_response
except HTTPError as e: # -G: changed `,` to `as` (required in Py3)
error = MWSError(str(e))
error.response = e.response
raise error
# Store the response object in the parsed_response for quick access
parsed_response.response = response
parsed_response.timestamp = now
# -G: MWS recommends saving metadata and timestamp.
# This makes it available in the parsed_response
return parsed_response
def download(link: str, name: str) -> None:
open(name, "wb").write(requests.request("GET", link).content)
# _*_coding:utf-8_*_ # @time :2020/11/30 6:03 下午 # @Author :hanjiping # Emil :[email protected] # File :gegister.py # @software PyCharm import requests from Common.INI_setting import config json_data = { "mobile_phone":"15011466717", "pwd":"12345678" } headers =eval(config.get('request','headers')) response = requests.request(method='post',headers = headers,json = json_data,url = "http://api.lemonban.com/futureloan/member/register" ) print(response.json())
import requests
url = "http://192.168.10.80/restconf/api/config/native/interface/Loopback/500"
payload = "{\r\n\t\"ned:Loopback\": {\r\n\t\t\"name\": 500,\r\n\t\t\"ip\": {\r\n\t\t\t\"address\": {\r\n\t\t\t\t\"primary\": {\r\n\t\t\t\t\t\"address\": \"170.99.1.1\",\r\n\t\t\t\t\t\"mask\": \"255.255.255.0\"\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n}"
headers = {
'Content-Type': "application/vnd.yang.data+json",
'Accept': "application/vnd.yang.data+json",
'Authorization': "Basic YWRtaW46Y2lzY28="
}
response = requests.request("PUT", url, data=payload, headers=headers)
print(response.text)
result.encoding = 'utf-8'
title = re.findall('<div class="zzj_3">(.*?)</div>', result.text)
datas = re.findall('<span class="zzj_f2">(.*?)</span>', result.text)
content = re.findall('<div class="zzj_5">(.*?)</div>', result.text,
re.DOTALL)
# print(title)
# print(datas)
# print(content)
return [
title[0], datas[0], datas[1], datas[2], datas[3].strip(), content[0]
]
# 首次采集分页总数
url = 'http://www16.zzu.edu.cn/msgs/vmsgisapi.dll/vmsglist?mtype=x&lan=202&tts=&tops=&pn=1'
result = requests.request('GET', url)
result.encoding = 'utf-8'
# print(re.findall('分(\d+)页', result.text))
# 获取总页数
page_num = int(re.findall('分(\d+)页', result.text)[0])
# print(get_list_urls(201))
i = 1 # 正在采集的数据
errorList = [] # 错误列表
while (True):
if page_num <= 0:
break
print('\033[1;31;40m正在采集第%d页\r\n' % (page_num, ))
print('*' * 50)
# 每一页的新闻链接
import json
import csv
port = ['ABMD.OQ']
k = len(port)
#===============================Pull Data=====================
url = "https://dev.api.thomsonreuters.com/eikon/v1/timeseries"
querystring = {"X-TR-API-APP-ID":"SVmwQ31EoclQ8QNgzN10QuXL57V8sRkO"}
payload = "{\r\n \"rics\": ['GOOG.O'],\r\n \"interval\": \"Daily\",\r\n \"startdate\": \"2010-01-04T00:00:00Z\",\r\n \"enddate\": \"2012-12-31T23:59:59Z\",\r\n \"fields\": \r\n [\"TIMESTAMP\",\"OPEN\",\"HIGH\",\"LOW\",\"CLOSE\",\"VOLUME\"]\r\n}"
headers = {
'cache-control': "no-cache",
'postman-token': "c5250829-c43b-8401-97d1-00f2095b4131"
}
resp1 = requests.request("POST", url, data=payload, headers=headers, params=querystring)
url = "https://dev.api.thomsonreuters.com/eikon/v1/datagrid"
querystring = {"X-TR-API-APP-ID":"SVmwQ31EoclQ8QNgzN10QuXL57V8sRkO"}
payload = "{\r\n \"instruments\": ['GOOG.O'],\r\n \"fields\":[ \r\n\t{\"name\": \"TR.GrossProfit.date\"},\r\n\t{\"name\": \"TR.GrossProfit\"},\r\n\t{\"name\": \"TR.OperatingIncome\"}, \r\n\t{\"name\": \"TR.NetIncomeBeforeTaxes\"},\r\n\t{\"name\": \"TR.TotalCurrentAssets\"}],\r\n \"parameters\": {\"Period\":\"FY-1\",\"Period\":\"FY0\"}\r\n}\r\n"
headers = {
'cache-control': "no-cache",
'postman-token': "b7a1a575-7d96-1ca7-a9dc-f55291d83e78"
}
resp2 = requests.request("POST", url, data=payload, headers=headers, params=querystring)
#===========================Assign data to lists======================
def start():
caller_phone_number = request.values.get('From')
user_id = request.values.get('CallSid')
twilio_asr_language = request.values.get('twilio_asr_language', "en-US")
apiai_language = request.values.get('apiai_language', "en")
caller_name = registered_users.get(caller_phone_number, " ")
hostname = request.url_root
# Initialize API.AI Bot
headers = {
'authorization': "Bearer " + apiai_client_access_key,
'content-type': "application/json"
}
payload = {
'event': {
'name': 'book_hotel_welcome',
'data': {
'user_name': caller_name
}
},
'lang': apiai_language,
'sessionId': user_id
}
response = requests.request("POST",
url=apiai_url,
data=json.dumps(payload),
headers=headers,
params=apiai_querystring)
print(response.text)
output = json.loads(response.text)
output_text = output['result']['fulfillment']['speech']
output_text = output_text.decode("utf-8")
resp = VoiceResponse()
# Prepare for next set of user Speech
values = {"prior_text": output_text}
qs = urllib.urlencode(values)
action_url = "/process_speech?" + qs
gather = Gather(input="speech",
hints=hints,
language=twilio_asr_language,
timeout="3",
action=action_url,
method="POST")
# TTS the bot response
qs = urllib.urlencode(values)
gather.say(output_text, voice='brian', language='en')
resp.append(gather)
# If gather is missing (no speech), redirect to process speech again
values = {
"prior_text": output_text,
"twilio_asr_language": twilio_asr_language,
"apiai_language": apiai_language,
"SpeechResult": "",
"Confidence": 0.0
}
qs = urllib.urlencode(values)
action_url = "/process_speech?" + qs
resp.redirect(action_url)
print str(resp)
return str(resp)
def run_all():
logger.info("**********本轮调度程序准备好了**********\n")
global pass_status, fail_reason, actual_result_text_re
variable_result_dict = {}
# 定义一个变量名与提取的结果字典
microservice_id_object = Microservice.objects.values_list("id").filter(
microservice_on_off="开").order_by("id")
# 查询微服务开关=开的微服务id
microservice_id_list = list(microservice_id_object)
microservice_id_list = list(chain.from_iterable(microservice_id_list))
data_object = Case.objects.values_list("id").filter(
case_on_off="开",
case_microservice__in=microservice_id_list).order_by("id")
# 反向查询微服务包含的用例信息
data_tuple = tuple(data_object)
# 把QuerySet对象转换成元祖
data_tuple = tuple(chain.from_iterable(data_tuple))
# 把多维元祖转换成一维元祖
for case_i in data_tuple:
case_object = Case.objects.get(id=case_i, case_on_off="开"). \
step_key.values().filter(step_on_off="开").order_by("id")
# 反向查询用例包含的步骤信息
case_list = list(case_object)
# 把QuerySet对象转换成列表
case_name = Case.objects.get(id=case_i, case_on_off="开").case_name
logger.info("用例名称为：{}".format(case_name))
service_domain = Case.objects.get(id=case_i, case_on_off="开"). \
case_microservice.microservice_environment.domain_name
dingtalk_webhook = Case.objects.get(id=case_i, case_on_off="开"). \
case_microservice.microservice_environment.webhook
secret = Case.objects.get(id=case_i, case_on_off="开"). \
case_microservice.microservice_environment.secret
recipient_email = Case.objects.get(id=case_i, case_on_off="开"). \
case_microservice.microservice_environment.recipient_email
switch_dict = model_to_dict(
Case.objects.get(id=case_i, case_on_off="开"),
fields=["dingtalk_on_off", "mailbox_on_off"])
dingtalk_switch = switch_dict["dingtalk_on_off"]
logger.info("钉钉开关为：{}".format(dingtalk_switch))
e_mail_switch = switch_dict["mailbox_on_off"]
logger.info("邮件开关为：{}".format(e_mail_switch))
for step_i in case_list:
step_id = step_i["id"]
step_name = step_i["step_name"]
step_on_off = step_i["step_on_off"]
request_mode = step_i["request_mode"]
api = step_i["api"]
body = step_i["body"]
headers = step_i["headers"]
query_string = step_i["query_string"]
expected_time = step_i["expected_time"]
expected_code = step_i["expected_code"]
expected_result = step_i["expected_result"]
regular = step_i["regular"]
if variable_result_dict:
if api:
api = function_dollar(api, variable_result_dict.items())
if body:
body = function_dollar(body, variable_result_dict.items())
if headers:
headers = function_dollar(headers,
variable_result_dict.items())
if query_string:
query_string = function_dollar(
query_string, variable_result_dict.items())
if expected_result:
expected_result = function_dollar(
expected_result, variable_result_dict.items())
if api:
api = function_rn(api)
api = function_rl(api)
if body:
body = function_rn(body)
body = function_rl(body)
body = function_mp(body)
body = function_rd(body)
body = json.loads(body)
if headers:
headers = function_rn(headers)
headers = function_rl(headers)
headers = function_mp(headers)
headers = function_rd(headers)
headers = json.loads(headers)
if query_string:
query_string = function_rn(query_string)
query_string = function_rl(query_string)
query_string = function_mp(query_string)
query_string = function_rd(query_string)
query_string = json.loads(query_string)
logger.info("步骤名称为：{}".format(step_name))
logger.info("请求方式为：{}".format(request_mode))
logger.info("步骤开关为：{}".format(step_on_off))
url = service_domain + api
logger.info("请求地址为：{}".format(url))
logger.info("请求头为：{}".format(
json.dumps(headers, ensure_ascii=False)))
logger.info("请求参数为：{}".format(
json.dumps(query_string, ensure_ascii=False)))
logger.info("请求体为：{}".format(json.dumps(body, ensure_ascii=False)))
try:
response = requests.request(request_mode,
url,
data=json.dumps(body),
headers=headers,
params=query_string,
timeout=(25, 30))
logger.info("HTTP请求成功")
except Exception as e:
logger.error("HTTP请求发生错误：{}".format(e))
if dingtalk_switch == "开":
alarm_message = http_request_exception_alarm(
case_name, step_name, url, request_mode,
json.dumps(body, ensure_ascii=False), "。。。。。。",
json.dumps(query_string, ensure_ascii=False), str(e))
send_dingtalk_alarm(alarm_message, dingtalk_webhook,
secret)
logger.info("钉钉报警发送成功")
if e_mail_switch == "开":
alarm_message = http_request_exception_alarm(
case_name, step_name, url, request_mode,
json.dumps(body, ensure_ascii=False), "。。。。。。",
json.dumps(query_string, ensure_ascii=False), str(e))
send_mailbox(alarm_message, recipient_email)
logger.info("邮件发送成功")
raise e
try:
actual_time = response.elapsed.total_seconds()
logger.info("实际的响应时间为：{}".format(actual_time))
except Exception as e:
logger.error("获取实际的响应时间发生错误：{}".format(e))
raise e
try:
actual_code = response.status_code
logger.info("实际的响应代码为：{}".format(actual_code))
except Exception as e:
logger.error("获取实际的响应代码发生错误：{}".format(e))
raise e
try:
actual_headers = response.headers
logger.info("实际的响应头为：{}".format(actual_headers))
except Exception as e:
logger.error("获取实际的响应头发生错误：{}".format(e))
raise e
try:
actual_result_text = response.text
logger.info("实际的响应结果为：{}".format(actual_result_text[0:400]))
except Exception as e:
logger.error("获取实际的响应结果发生错误：{}".format(e))
raise e
try:
if regular:
regular = demjson.decode(regular)
extract_list = []
for i in regular["expression"]:
re_list = re.findall(i, actual_result_text)
if len(re_list) >= 1:
regular_result = re_list[0]
extract_list.append(regular_result)
variable_result_dict_temporary = dict(
zip(regular["variable"], extract_list))
for key, value in variable_result_dict_temporary.items():
variable_result_dict[key] = value
except Exception as e:
logger.error("正则匹配发生错误：{}".format(e))
raise e
if variable_result_dict:
for key in list(variable_result_dict.keys()):
if not variable_result_dict[key]:
del variable_result_dict[key]
expected_result = re.sub("{|}|\'|\"|\\[|\\]| ", "",
expected_result)
if actual_result_text:
actual_result_text_re = re.sub("{|}|\'|\"|\\[|\\]| ", "",
actual_result_text)
expected_result_list = re.split(":|,", expected_result)
actual_result_list_sp = re.split(":|,", actual_result_text_re)
if expected_code == actual_code:
fail_reason = ""
if set(expected_result_list) <= set(actual_result_list_sp):
pass_status = "是"
if expected_time:
if actual_time <= expected_time:
pass_status = "是"
else:
pass_status = "否"
fail_reason = "实际的响应时间大于预期的响应时间"
logger.error("实际的响应时间大于预期的响应时间")
if dingtalk_switch == "开":
alarm_message = response_time_alarm(
case_name, step_name, url, request_mode,
json.dumps(body,
ensure_ascii=False), "。。。。。。",
json.dumps(query_string,
ensure_ascii=False),
actual_time, expected_time)
send_dingtalk_alarm(alarm_message,
dingtalk_webhook, secret)
logger.info("钉钉报警发送成功")
if e_mail_switch == "开":
alarm_message = response_time_alarm(
case_name, step_name, url, request_mode,
json.dumps(body,
ensure_ascii=False), "。。。。。。",
json.dumps(query_string,
ensure_ascii=False),
actual_time, expected_time)
send_mailbox(alarm_message, recipient_email)
logger.info("邮件发送成功")
else:
pass_status = "否"
fail_reason = "预期的响应结果与实际的响应结果断言失败"
logger.error("预期的响应结果与实际的响应结果断言失败")
if dingtalk_switch == "开":
alarm_message = response_result_alarm(
case_name,
step_name,
url,
request_mode,
json.dumps(body, ensure_ascii=False),
"。。。。。。",
json.dumps(query_string, ensure_ascii=False),
)
send_dingtalk_alarm(alarm_message, dingtalk_webhook,
secret)
logger.info("钉钉报警发送成功")
if e_mail_switch == "开":
alarm_message = response_result_alarm(
case_name,
step_name,
url,
request_mode,
json.dumps(body, ensure_ascii=False),
"。。。。。。",
json.dumps(query_string, ensure_ascii=False),
)
send_mailbox(alarm_message, recipient_email)
logger.info("邮件发送成功")
else:
pass_status = "否"
fail_reason = "预期的响应代码与实际的响应代码不相等"
logger.error("预期的响应代码与实际的响应代码不相等")
if dingtalk_switch == "开":
alarm_message = response_code_alarm(
case_name, step_name, url, request_mode,
json.dumps(body, ensure_ascii=False), "。。。。。。",
json.dumps(query_string, ensure_ascii=False),
expected_code, actual_code)
send_dingtalk_alarm(alarm_message, dingtalk_webhook,
secret)
logger.info("钉钉报警发送成功")
if e_mail_switch == "开":
alarm_message = response_code_alarm(
case_name, step_name, url, request_mode,
json.dumps(body, ensure_ascii=False), "。。。。。。",
json.dumps(query_string, ensure_ascii=False),
expected_code, actual_code)
send_mailbox(alarm_message, recipient_email)
logger.info("邮件发送成功")
RunningResults.objects.create(
running_results_step_id=step_id,
pass_status=pass_status,
fail_reason=fail_reason,
run_time=datetime.now(),
actual_time=actual_time,
actual_code=actual_code,
actual_result=actual_result_text,
)
logger.info("**********本轮调度程序已经完成，请等待下一轮**********\n")
def _action(self):
self._result = requests.request(self.method, self.url, data=self.query).json()
def external_service_request(endpoint,
data_json=None,
session=None,
method='POST',
headers=None,
service_access=None,
timeout=5,
**kwargs):
"""Sends request to external service"""
# pylint: disable=too-many-branches, disable=too-many-nested-blocks
default_headers = {
'Content-type': 'application/json',
'Accept': 'application/json'
}
if headers and isinstance(headers, dict):
default_headers.update(headers)
tries = 0
while True:
if tries >= CFG.request_retries:
break
try:
if session:
response = session.request(method=method,
url=endpoint,
headers=default_headers,
json=data_json,
timeout=timeout,
**kwargs)
else:
response = requests.request(method=method,
url=endpoint,
headers=default_headers,
json=data_json,
timeout=timeout,
**kwargs)
if response.status_code == 200:
if service_access is not None:
service_access.status = True
return response.json()
if response.status_code >= 500:
if service_access is not None:
service_access.status = False
LOGGER.info(
"External service - %s temporarily unavailable.",
endpoint)
break
LOGGER.info(
"External service - %s temporarily unavailable, retrying...",
endpoint)
sleep(1)
else:
tries += 1
VMAAS_RETURN_ERR.inc()
LOGGER.error(
"Error during %s request to endpoint %s: HTTP %s, %s",
method, endpoint, response.status_code, response.text)
LOGGER.debug("JSON: %s", str(data_json))
# Do not retry for 4xx HTTP codes
if 400 <= response.status_code < 500:
if service_access is not None:
if response.status_code == 404:
service_access.status = True
else:
service_access.status = False
break
except requests.exceptions.Timeout:
VMAAS_CNX_ERR.inc()
LOGGER.exception("Timeout error calling external service: %s: ",
endpoint)
break
except requests.exceptions.RequestException:
tries += 1
VMAAS_CNX_ERR.inc()
LOGGER.exception("Error calling external service %s: ", endpoint)
return None
def test_post(mock_requests):
print(requests.request("get", "http://www.baidu.com"))
print(requests.post("http://www.baidu.com"))
def http_request(method,
url_suffix,
params=None,
data=None,
headers=HEADERS,
safe=False,
get_token_flag=True):
"""
A wrapper for requests lib to send our requests and handle requests and responses better.
:type method: ``str``
:param method: HTTP method for the request.
:type url_suffix: ``str``
:param url_suffix: The suffix of the URL (endpoint)
:type params: ``dict``
:param params: The URL params to be passed.
:type data: ``str``
:param data: The body data of the request.
:type headers: ``dict``
:param headers: Request headers
:type safe: ``bool``
:param safe: If set to true will return None in case of http error
:type get_token_flag: ``bool``
:param get_token_flag: If set to True will call get_token()
:return: Returns the http request response json
:rtype: ``dict``
"""
if get_token_flag:
token = get_token()
headers['Authorization'] = 'Bearer {}'.format(token)
url = SERVER + url_suffix
try:
res = requests.request(
method,
url,
verify=USE_SSL,
params=params,
data=data,
headers=headers,
)
except requests.exceptions.RequestException:
return_error(
'Error in connection to the server. Please make sure you entered the URL correctly.'
)
# Handle error responses gracefully
if res.status_code not in {200, 201, 202}:
err_msg = 'Error in API call. code:{code}; reason: {reason}'.format(
code=res.status_code, reason=res.reason)
# try to create a new token
if res.status_code == 403 and get_token_flag:
LOG(err_msg)
token = get_token(new_token=True)
headers['Authorization'] = 'Bearer {}'.format(token)
return http_request(method,
url_suffix,
params,
data,
headers,
safe,
get_token_flag=False)
elif safe:
return None
return_error(err_msg)
return res.json()
def download(id):
apiToken = "8NUQ1BwOxdONDWTtUXIaqXIQ0ka582wdEOJ2XkSv"
surveyId = id
fileFormat = "json"
dataCenter = 'co1'
# Setting static parameters
requestCheckProgress = 0
progressStatus = "in progress"
baseUrl = "https://{0}.qualtrics.com/API/v3/responseexports/".format(
dataCenter)
headers = {
"content-type": "application/json",
"x-api-token": apiToken,
}
# Step 1: Creating Data Export
downloadRequestUrl = baseUrl
downloadRequestPayload = '{"format":"' + fileFormat + '","surveyId":"' + surveyId + '"}'
downloadRequestResponse = requests.request("POST",
downloadRequestUrl,
data=downloadRequestPayload,
headers=headers)
progressId = downloadRequestResponse.json()["result"]["id"]
print(downloadRequestResponse.text)
# Step 2: Checking on Data Export Progress and waiting until export is ready
isFile = None
while requestCheckProgress < 100 and progressStatus is not "complete" and isFile is None:
requestCheckUrl = baseUrl + progressId
requestCheckResponse = requests.request("GET",
requestCheckUrl,
headers=headers)
isFile = (requestCheckResponse.json()["result"]["file"])
if isFile is None:
print("file not ready")
else:
print("file created:",
requestCheckResponse.json()["result"]["file"])
requestCheckProgress = requestCheckResponse.json(
)["result"]["percentComplete"]
print("Download is " + str(requestCheckProgress) + " complete")
# Step 3: Downloading file
requestDownloadUrl = baseUrl + progressId + '/file'
requestDownload = requests.request("GET",
requestDownloadUrl,
headers=headers,
stream=True)
regex = r"([a-zA-Z0-9,.\-|;!_?\[\]&\ \(\)]+.json)"
text = str(requestDownload.content, 'utf-8', errors='replace')
filename = re.findall(regex, text)
# Step 4: Unzipping the file
zipfile.ZipFile(io.BytesIO(
requestDownload.content)).extractall("MyQualtricsDownload")
# os.chdir(os.path.join(os.path.join(os.path.expanduser('~')), 'Desktop'))
dir = os.getcwd()
# open(os.mkdir(dir + "/MyQualtricsDownload/") + str(filename[0]), 'w')
with open(str(dir) + "/MyQualtricsDownload/" + str(filename[0]),
'r') as json_file:
json_decoded = json.load(json_file)
for element in json_decoded['responses']:
element['surveyID'] = id
element['surveyName'] = str(filename[0])
with open(str(dir) + "/MyQualtricsDownload/" + str(filename[0]),
'w') as json_out_file:
json.dump(json_decoded,
json_out_file,
indent=4,
separators=(',', ': '))
print('Complete')
isExists_path = os.path.exists(
os.path.join(os.path.join(os.path.expanduser('~')), 'Desktop') +
"/MyQualtricsDownload/")
if not isExists_path:
os.mkdir(
os.path.join(os.path.join(os.path.expanduser('~')), 'Desktop') +
"/MyQualtricsDownload/")
isExists_file = os.path.exists(
os.path.join(os.path.join(os.path.expanduser('~')), 'Desktop') +
"/MyQualtricsDownload/" + str(filename[0]))
if isExists_file:
os.remove(
os.path.join(os.path.join(os.path.expanduser('~')), 'Desktop') +
"/MyQualtricsDownload/" + str(filename[0]))
shutil.move(
(str(dir) + "/MyQualtricsDownload/" + str(filename[0])),
(os.path.join(os.path.join(os.path.expanduser('~')), 'Desktop') +
"/MyQualtricsDownload/"))
def test_mock():
requests.request=MagicMock(return_value="mock")
print(requests.request("get", "http://www.baidu.com"))
# define where we're going to save the data
path_save_data = "data/example_data"
filename_save_topics = "{:s}/augmento_topics.msgpack.zlib".format(
path_save_data)
filename_save_sources = "{:s}/augmento_sources.msgpack.zlib".format(
path_save_data)
filename_save_coins = "{:s}/augmento_coins.msgpack.zlib".format(path_save_data)
filename_save_bin_sizes = "{:s}/augmento_bin_sizes.msgpack.zlib".format(
path_save_data)
# check if the data path exists
ioh.check_path(path_save_data, create_if_not_exist=True)
# save a list of the augmento topics
r = requests.request("GET", topics_endpoint_url, timeout=10)
print("saving topics to {:s}".format(filename_save_topics))
with open(filename_save_topics, "wb") as f:
f.write(zlib.compress(msgpack.packb(r.json())))
# save a list of the augmento topics
r = requests.request("GET", sources_endpoint_url, timeout=10)
print("saving sources to {:s}".format(filename_save_sources))
with open(filename_save_sources, "wb") as f:
f.write(zlib.compress(msgpack.packb(r.json())))
# save a list of the augmento topics
r = requests.request("GET", coins_endpoint_url, timeout=10)
print("saving coins to {:s}".format(filename_save_coins))
with open(filename_save_coins, "wb") as f:
f.write(zlib.compress(msgpack.packb(r.json())))
import sys
from utils.util import getRootPath
import requests
from bs4 import BeautifulSoup
import multiprocessing
from contextlib import closing
from tqdm import tqdm
import requests
import progressbar
import requests.packages.urllib3
save_path="./news1.mp4"
url = "https://dns.63mimi.com/20180126/10/1/xml/91_d1f327803d104889af78aadd68c069fe.mp4"
response = requests.request("GET", url, stream=True, data=None, headers=None)
total_length = int(response.headers.get("Content-Length"))
with open(save_path, 'wb') as f:
# widgets = ['Processed: ', progressbar.Counter(), ' lines (', progressbar.Timer(), ')']
# pbar = progressbar.ProgressBar(widgets=widgets)
# for chunk in pbar((i for i in response.iter_content(chunk_size=1))):
# if chunk:
# f.write(chunk)
# f.flush()
widgets = ['Progress: ', progressbar.Percentage(), ' ',
progressbar.Bar(marker='#', left='[', right=']'),
' ', progressbar.ETA(), ' ', progressbar.FileTransferSpeed()]
pbar = progressbar.ProgressBar(widgets=widgets, maxval=total_length).start()
for chunk in response.iter_content(chunk_size=1):
def cancelIllegalOrders():
for order in illegal:
split = order.split(":")
orderID = split[0]
url = "https://api.bigcommerce.com/stores/" + str(
storehash) + "/v2/orders/" + str(orderID)
response = requests.request("GET", url, headers=headers)
get = response.json()
cancelList[orderID] = get['status']
fullName = str(get['billing_address']['first_name']) + " " + (str(
get['billing_address']['last_name']))
for canceled in cancelList:
if response.status_code != 200:
log_print(
"ERROR WHEN TRYING TO ENACT PUT METHOD FOR ORDER CANCELING! REPORT TO ADMIN IF YOU SEE THIS."
)
return
if (get['status'] == "Awaiting Payment"
or get['status'] == "Awaiting Fulfillment"):
url2 = "https://api.bigcommerce.com/stores/" + str(
storehash) + "/v2/orders/" + str(canceled)
payload2 = "{\"status_id\":5}"
response2 = requests.request("PUT",
url2,
data=payload2,
headers=headers)
json_data2 = response2.json()
currentCaneclation = badOrderObj(orderID, fullName, split[1],
split[2])
log_print(
"NEW OBJECT FOR CUSTOMER/ORDER: {} : {} : {} : {}".format(
currentCaneclation.getOrderID(),
currentCaneclation.getName(),
currentCaneclation.getZipcode(),
currentCaneclation.getIllegalItem()))
key = "{}:{}".format(orderID, get['billing_address']['email'])
illegalObjects[key] = currentCaneclation
log_print("\nOrder [{}] was canceled!\n".format(orderID))
result[
canceled] = "Status was [{}]. Requirments fulfilled and ORDER_ID [{}] canceled! Confirmtaion: NEW_ORDER_STATUS[{}]".format(
cancelList[canceled], canceled, json_data2['status'])
else:
url = "https://api.bigcommerce.com/stores/" + str(
storehash) + "/v2/orders/" + str(canceled)
response = requests.request("GET", url, headers=headers)
get2 = response.json()
cancelList[orderID] = get['status']
result[
canceled] = "Status was [{}]. Requirments FAILED to be fulfilled as ORDER_ID [{}]'s status is [{}], which means we cannot cancel!".format(
cancelList[canceled], canceled, get2['status'])
for x in result:
log_print("Cancelation Status: {} Result: {}".format(
x, result[x]))
def get_data(tup):
global g_id_cache
global g_count
cid = tup[0]
cname = tup[2]
url = "http://it.snssdk.com/api/news/feed/v63/"
t = int(time.time() / 10000)
t = random.randint(6 * t, 10 * t)
querystring = {
"category": cname,
"concern_id": "6215497896830175745",
"refer": "1",
"count": "20",
"max_behot_time": t,
"last_refresh_sub_entrance_interval": "1524907088",
"loc_mode": "5",
"tt_from": "pre_load_more",
"cp": "51a5ee4f38c50q1",
"plugin_enable": "0",
"iid": "31047425023",
"device_id": "51425358841",
"ac": "wifi",
"channel": "tengxun",
"aid": "13",
"app_name": "news_article",
"version_code": "631",
"version_name": "6.3.1",
"device_platform": "android",
"ab_version":
"333116,297979,317498,336556,295827,325046,239097,324283,170988,335432,332098,325198,336443,330632,297058,276203,286212,313219,328615,332041,329358,322321,327537,335710,333883,335102,334828,328670,324007,317077,334305,280773,335671,319960,333985,331719,336452,214069,31643,332881,333968,318434,207253,266310,321519,247847,281298,328218,335998,325618,333327,336199,323429,287591,288418,260650,326188,324614,335477,271178,326588,326524,326532",
"ab_client": "a1,c4,e1,f2,g2,f7",
"ab_feature": "94563,102749",
"abflag": "3",
"ssmix": "a",
"device_type": "MuMu",
"device_brand": "Android",
"language": "zh",
"os_api": "19",
"os_version": "4.4.4",
"uuid": "008796762094657",
"openudid": "b7215ea70ca32066",
"manifest_version_code": "631",
"resolution": "1280*720",
"dpi": "240",
"update_version_code": "6310",
"_rticket": "1524907088018",
"plugin": "256"
}
headers = {
'cache-control': "no-cache",
'postman-token': "26530547-e697-1e8b-fd82-7c6014b3ee86",
'User-Agent': g_ua
}
response = requests.request("GET",
url,
headers=headers,
params=querystring)
jj = json.loads(response.text)
with open('toutiao_cat_data.txt', 'a') as fp:
for item in jj['data']:
item = item['content']
item = item.replace('\"', '"')
# print item
# item = item.decode('utf-8')
item = json.loads(item)
kws = ''
if item.has_key('keywords'):
kws = item['keywords']
if item.has_key('ad_id'):
print 'ad'
elif not item.has_key('item_id') or not item.has_key('title'):
print 'bad'
else:
item_id = item['item_id']
print g_count, cid, cname, item['item_id'], item['title'], kws
if g_id_cache.has_key(item_id):
print 'dulp'
else:
g_id_cache[item_id] = 1
line = u"{}_!_{}_!_{}_!_{}_!_{}".format(
item['item_id'], cid, cname, item['title'], kws)
line = line.replace('\n', '').replace('\r', '')
line = line + '\n'
fp.write(line.encode('utf-8'))
g_count += 1
url = f"https://{hostname}:9060/ers/config/endpoint"
payload = {
"ERSEndPoint": {
"name": endpoint_name,
"description": endpoint_description,
"mac": endpoint_mac,
"groupId": endpoint_group
}
}
headers = {
'content-type': "application/json",
'accept': "application/json"
}
try:
response = requests.request("POST",
url,
data=json.dumps(payload),
headers=headers,
auth=(username, password),
verify=False)
if (response.status_code == 201):
print(f"User {endpoint_name} successfully added to group.")
else:
print(f"Error {response.status_code} returned from ISE.")
except:
response.raise_for_status()
lst = open(sys.argv[1], 'r')
while True:
sc = open(sys.argv[2]).read()
suc = open('Result.txt', 'a')
host = lst.readline().replace('\n', '')
if not host:
break
if not host.startswith('http'):
host = 'http://' + host
out = '/' + sys.argv[2]
print('\t [*] Filename : ' + sys.argv[2])
print('\t [*] Target : ' + host)
try:
r = requests.request(
'put',
host + out,
data=sc,
headers={'Content-Type': 'application/octet-stream'})
except:
print('\t Failed : Null Respone\n')
pass
continue
if r.status_code == 200:
print('\t [*] Sukses : ' + host + out + '\n')
suc.write(host + out + '\n')
else:
print('\t \033[91m[!] Failed : ' + host + out + '\n')
print("\t \033[34;1m[*]\033[0m Output Saved On : Result.txt")