def test_string(self):
""" Retry string representation looks the way we expect """
retry = Retry()
self.assertEqual(str(retry), 'Retry(total=10, connect=None, read=None, redirect=None)')
for _ in range(3):
retry = retry.increment()
self.assertEqual(str(retry), 'Retry(total=7, connect=None, read=None, redirect=None)')
def test_retry_higher_total_loses(self):
""" A lower connect timeout than the total is honored """
error = ConnectTimeoutError()
retry = Retry(connect=2, total=3)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
self.assertRaises(MaxRetryError, retry.increment, error=error)
def test_retry_read_zero(self):
""" No second chances on read timeouts, by default """
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry(read=0)
with pytest.raises(MaxRetryError) as e:
retry.increment(method='GET', error=error)
assert e.value.reason == error
def test_retry_higher_total_loses_vs_read(self):
""" A lower read timeout than the total is honored """
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry(read=2, total=3)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
self.assertRaises(MaxRetryError, retry.increment, error=error)
def test_string(self):
""" Retry string representation looks the way we expect """
retry = Retry()
assert str(retry) == 'Retry(total=10, connect=None, read=None, redirect=None, status=None)'
for _ in range(3):
retry = retry.increment(method='GET')
assert str(retry) == 'Retry(total=7, connect=None, read=None, redirect=None, status=None)'
def test_retry_higher_total_loses_vs_read(self):
""" A lower read timeout than the total is honored """
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry(read=2, total=3)
retry = retry.increment(method='GET', error=error)
retry = retry.increment(method='GET', error=error)
with pytest.raises(MaxRetryError):
retry.increment(method='GET', error=error)
def test_status_counter(self):
resp = HTTPResponse(status=400)
retry = Retry(status=2)
retry = retry.increment(response=resp)
retry = retry.increment(response=resp)
with pytest.raises(MaxRetryError) as e:
retry.increment(response=resp)
assert str(e.value.reason) == ResponseError.SPECIFIC_ERROR.format(status_code=400)
def test_retry_both_specified(self):
"""Total can win if it's lower than the connect value"""
error = ConnectTimeoutError()
retry = Retry(connect=3, total=2)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
with pytest.raises(MaxRetryError) as e:
retry.increment(error=error)
assert e.value.reason == error
def test_retry_read_zero(self):
""" No second chances on read timeouts, by default """
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry(read=0)
try:
retry.increment(error=error)
self.fail("Failed to raise error.")
except MaxRetryError as e:
self.assertEqual(e.reason, error)
def test_retry_both_specified(self):
"""Total can win if it's lower than the connect value"""
error = ConnectTimeoutError()
retry = Retry(connect=3, total=2)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
try:
retry.increment(error=error)
self.fail("Failed to raise error.")
except MaxRetryError as e:
self.assertEqual(e.reason, error)
def test_error_message(self):
retry = Retry(total=0)
with pytest.raises(MaxRetryError) as e:
retry = retry.increment(method='GET',
error=ReadTimeoutError(None, "/", "read timed out"))
assert 'Caused by redirect' not in str(e.value)
assert str(e.value.reason) == 'None: read timed out'
retry = Retry(total=1)
with pytest.raises(MaxRetryError) as e:
retry = retry.increment('POST', '/')
retry = retry.increment('POST', '/')
assert 'Caused by redirect' not in str(e.value)
assert isinstance(e.value.reason, ResponseError)
assert str(e.value.reason) == ResponseError.GENERIC_ERROR
retry = Retry(total=1)
response = HTTPResponse(status=500)
with pytest.raises(MaxRetryError) as e:
retry = retry.increment('POST', '/', response=response)
retry = retry.increment('POST', '/', response=response)
assert 'Caused by redirect' not in str(e.value)
msg = ResponseError.SPECIFIC_ERROR.format(status_code=500)
assert str(e.value.reason) == msg
retry = Retry(connect=1)
with pytest.raises(MaxRetryError) as e:
retry = retry.increment(error=ConnectTimeoutError('conntimeout'))
retry = retry.increment(error=ConnectTimeoutError('conntimeout'))
assert 'Caused by redirect' not in str(e.value)
assert str(e.value.reason) == 'conntimeout'
def test_history(self):
retry = Retry(total=10)
self.assertEqual(retry.history, tuple())
connection_error = ConnectTimeoutError('conntimeout')
retry = retry.increment('GET', '/test1', None, connection_error)
self.assertEqual(retry.history, (RequestHistory('GET', '/test1', connection_error, None, None),))
read_error = ReadTimeoutError(None, "/test2", "read timed out")
retry = retry.increment('POST', '/test2', None, read_error)
self.assertEqual(retry.history, (RequestHistory('GET', '/test1', connection_error, None, None),
RequestHistory('POST', '/test2', read_error, None, None)))
response = HTTPResponse(status=500)
retry = retry.increment('GET', '/test3', response, None)
self.assertEqual(retry.history, (RequestHistory('GET', '/test1', connection_error, None, None),
RequestHistory('POST', '/test2', read_error, None, None),
RequestHistory('GET', '/test3', None, 500, None)))
def test_parse_retry_after(self):
invalid = [
"-1",
"+1",
"1.0",
six.u("\xb2"), # \xb2 = ^2
]
retry = Retry()
for value in invalid:
self.assertRaises(InvalidHeader, retry.parse_retry_after, value)
self.assertEqual(retry.parse_retry_after("0"), 0)
self.assertEqual(retry.parse_retry_after("1000"), 1000)
self.assertEqual(retry.parse_retry_after("\t42 "), 42)
def test_status_forcelist(self):
retry = Retry(status_forcelist=xrange(500,600))
self.assertFalse(retry.is_forced_retry('GET', status_code=200))
self.assertFalse(retry.is_forced_retry('GET', status_code=400))
self.assertTrue(retry.is_forced_retry('GET', status_code=500))
retry = Retry(total=1, status_forcelist=[418])
self.assertFalse(retry.is_forced_retry('GET', status_code=400))
self.assertTrue(retry.is_forced_retry('GET', status_code=418))
# String status codes are not matched.
retry = Retry(total=1, status_forcelist=['418'])
self.assertFalse(retry.is_forced_retry('GET', status_code=418))
def test_history(self):
retry = Retry(total=10, method_whitelist=frozenset(['GET', 'POST']))
assert retry.history == tuple()
connection_error = ConnectTimeoutError('conntimeout')
retry = retry.increment('GET', '/test1', None, connection_error)
history = (RequestHistory('GET', '/test1', connection_error, None, None),)
assert retry.history == history
read_error = ReadTimeoutError(None, "/test2", "read timed out")
retry = retry.increment('POST', '/test2', None, read_error)
history = (RequestHistory('GET', '/test1', connection_error, None, None),
RequestHistory('POST', '/test2', read_error, None, None))
assert retry.history == history
response = HTTPResponse(status=500)
retry = retry.increment('GET', '/test3', response, None)
history = (RequestHistory('GET', '/test1', connection_error, None, None),
RequestHistory('POST', '/test2', read_error, None, None),
RequestHistory('GET', '/test3', None, 500, None))
assert retry.history == history
def test_status_forcelist(self):
retry = Retry(status_forcelist=xrange(500, 600))
assert not retry.is_retry('GET', status_code=200)
assert not retry.is_retry('GET', status_code=400)
assert retry.is_retry('GET', status_code=500)
retry = Retry(total=1, status_forcelist=[418])
assert not retry.is_retry('GET', status_code=400)
assert retry.is_retry('GET', status_code=418)
# String status codes are not matched.
retry = Retry(total=1, status_forcelist=['418'])
assert not retry.is_retry('GET', status_code=418)
def test_backoff_reset_after_redirect(self):
retry = Retry(total=100, redirect=5, backoff_factor=0.2)
retry = retry.increment(method='GET')
retry = retry.increment(method='GET')
assert retry.get_backoff_time() == 0.4
redirect_response = HTTPResponse(status=302, headers={'location': 'test'})
retry = retry.increment(method='GET', response=redirect_response)
assert retry.get_backoff_time() == 0
retry = retry.increment(method='GET')
retry = retry.increment(method='GET')
assert retry.get_backoff_time() == 0.4
def test_backoff_reset_after_redirect(self):
retry = Retry(total=100, redirect=5, backoff_factor=0.2)
retry = retry.increment()
retry = retry.increment()
self.assertEqual(retry.get_backoff_time(), 0.4)
redirect_response = HTTPResponse(status=302, headers={'location': 'test'})
retry = retry.increment(response=redirect_response)
self.assertEqual(retry.get_backoff_time(), 0)
retry = retry.increment()
retry = retry.increment()
self.assertEqual(retry.get_backoff_time(), 0.4)
def test_retry_total_none(self):
""" if Total is none, connect error should take precedence """
error = ConnectTimeoutError()
retry = Retry(connect=2, total=None)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
with pytest.raises(MaxRetryError) as e:
retry.increment(error=error)
assert e.value.reason == error
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry(connect=2, total=None)
retry = retry.increment(method='GET', error=error)
retry = retry.increment(method='GET', error=error)
retry = retry.increment(method='GET', error=error)
assert not retry.is_exhausted()
def test_status_forcelist(self):
retry = Retry(status_forcelist=xrange(500,600))
self.assertFalse(retry.is_forced_retry('GET', status_code=200))
self.assertFalse(retry.is_forced_retry('GET', status_code=400))
self.assertTrue(retry.is_forced_retry('GET', status_code=500))
retry = Retry(total=1, status_forcelist=[418])
self.assertFalse(retry.is_forced_retry('GET', status_code=400))
self.assertTrue(retry.is_forced_retry('GET', status_code=418))
def test_method_whitelist_with_status_forcelist(self):
# Falsey method_whitelist means to retry on any method.
retry = Retry(status_forcelist=[500], method_whitelist=None)
self.assertTrue(retry.is_forced_retry('GET', status_code=500))
self.assertTrue(retry.is_forced_retry('POST', status_code=500))
# Criteria of method_whitelist and status_forcelist are ANDed.
retry = Retry(status_forcelist=[500], method_whitelist=['POST'])
self.assertFalse(retry.is_forced_retry('GET', status_code=500))
self.assertTrue(retry.is_forced_retry('POST', status_code=500))
def test_retry_total_none(self):
""" if Total is none, connect error should take precedence """
error = ConnectTimeoutError()
retry = Retry(connect=2, total=None)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
try:
retry.increment(error=error)
self.fail("Failed to raise error.")
except MaxRetryError as e:
self.assertEqual(e.reason, error)
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry(connect=2, total=None)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
self.assertFalse(retry.is_exhausted())
def __init__(self, pool_connections=DEFAULT_POOLSIZE,
pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES,
pool_block=DEFAULT_POOLBLOCK):
if max_retries == DEFAULT_RETRIES:
self.max_retries = Retry(0, read=False)
else:
self.max_retries = Retry.from_int(max_retries)
self.config = {}
super(HTTP2Adapter, self).__init__()
self._pool_connections = pool_connections
self._pool_maxsize = pool_maxsize
self._pool_block = pool_block
self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
def test_retry_default(self):
""" If no value is specified, should retry connects 3 times """
retry = Retry()
assert retry.total == 10
assert retry.connect is None
assert retry.read is None
assert retry.redirect is None
error = ConnectTimeoutError()
retry = Retry(connect=1)
retry = retry.increment(error=error)
with pytest.raises(MaxRetryError):
retry.increment(error=error)
retry = Retry(connect=1)
retry = retry.increment(error=error)
assert not retry.is_exhausted()
assert Retry(0).raise_on_redirect
assert not Retry(False).raise_on_redirect
def test_error_message(self):
retry = Retry(total=0)
try:
retry = retry.increment(method='GET',
error=ReadTimeoutError(None, "/", "read timed out"))
raise AssertionError("Should have raised a MaxRetryError")
except MaxRetryError as e:
assert 'Caused by redirect' not in str(e)
self.assertEqual(str(e.reason), 'None: read timed out')
retry = Retry(total=1)
try:
retry = retry.increment('POST', '/')
retry = retry.increment('POST', '/')
raise AssertionError("Should have raised a MaxRetryError")
except MaxRetryError as e:
assert 'Caused by redirect' not in str(e)
self.assertTrue(isinstance(e.reason, ResponseError),
"%s should be a ResponseError" % e.reason)
self.assertEqual(str(e.reason), ResponseError.GENERIC_ERROR)
retry = Retry(total=1)
try:
response = HTTPResponse(status=500)
retry = retry.increment('POST', '/', response=response)
retry = retry.increment('POST', '/', response=response)
raise AssertionError("Should have raised a MaxRetryError")
except MaxRetryError as e:
assert 'Caused by redirect' not in str(e)
msg = ResponseError.SPECIFIC_ERROR.format(status_code=500)
self.assertEqual(str(e.reason), msg)
retry = Retry(connect=1)
try:
retry = retry.increment(error=ConnectTimeoutError('conntimeout'))
retry = retry.increment(error=ConnectTimeoutError('conntimeout'))
raise AssertionError("Should have raised a MaxRetryError")
except MaxRetryError as e:
assert 'Caused by redirect' not in str(e)
self.assertEqual(str(e.reason), 'conntimeout')
def test_error_message(self):
retry = Retry(total=0)
try:
retry = retry.increment(error=ReadTimeoutError(None, "/", "read timed out"))
raise AssertionError("Should have raised a MaxRetryError")
except MaxRetryError as e:
assert 'Caused by redirect' not in str(e)
self.assertEqual(str(e.reason), 'None: read timed out')
retry = Retry(total=1)
try:
retry = retry.increment('POST', '/')
retry = retry.increment('POST', '/')
raise AssertionError("Should have raised a MaxRetryError")
except MaxRetryError as e:
assert 'Caused by redirect' not in str(e)
self.assertEqual(str(e.reason), 'received erroneous response too many times')
retry = Retry(total=1)
try:
response = HTTPResponse(status=500)
retry = retry.increment('POST', '/', response=response)
retry = retry.increment('POST', '/', response=response)
raise AssertionError("Should have raised a MaxRetryError")
except MaxRetryError as e:
assert 'Caused by redirect' not in str(e)
self.assertEqual(str(e.reason), 'received 500 server error too many times')
retry = Retry(connect=1)
try:
retry = retry.increment(error=ConnectTimeoutError('conntimeout'))
retry = retry.increment(error=ConnectTimeoutError('conntimeout'))
raise AssertionError("Should have raised a MaxRetryError")
except MaxRetryError as e:
assert 'Caused by redirect' not in str(e)
self.assertEqual(str(e.reason), 'conntimeout')
def test_retry_default(self):
""" If no value is specified, should retry connects 3 times """
retry = Retry()
self.assertEqual(retry.total, 10)
self.assertEqual(retry.connect, None)
self.assertEqual(retry.read, None)
self.assertEqual(retry.redirect, None)
error = ConnectTimeoutError()
retry = Retry(connect=1)
retry = retry.increment(error=error)
self.assertRaises(MaxRetryError, retry.increment, error=error)
retry = Retry(connect=1)
retry = retry.increment(error=error)
self.assertFalse(retry.is_exhausted())
self.assertTrue(Retry(0).raise_on_redirect)
self.assertFalse(Retry(False).raise_on_redirect)
def call_dividend_api():
from airflow.operators.python_operator import PythonOperator
from airflow.operators.bash_operator import BashOperator
from airflow.hooks.base_hook import BaseHook
from airflow.models import DAG
from airflow.utils import dates
from datetime import datetime, timedelta
from google.cloud import bigquery
from google.oauth2 import service_account
import requests
import json
import pandas_gbq as pdgbq
from datetime import date
from datetime import datetime, timedelta
import numpy as np
import pandas as pd
import json
import os
import time
from airflow.models import Variable
from urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
from pandas.api.types import is_datetime64_any_dtype as is_datetime
from sendgrid import SendGridAPIClient
from sendgrid.helpers.mail import Mail
from string import Template
from sendgrid.helpers.mail import To, Attachment, FileContent, FileType, FileName, Disposition, ContentId
import base64
import pickle
credentials = service_account.Credentials.from_service_account_info(
Variable.get("key", deserialize_json=True))
project_id = 'hackathon-wpb'
table_id = 'hackathon-wpb.customer_relations.customer_dividend_malayasia'
query_string = """
SELECT *
FROM hackathon-wpb.customer_relations.customer_dividend_malayasia"""
url = "https://globalhistorical.xignite.com/v3/xGlobalHistorical.json/GetCashDividendHistory"
client = bigquery.Client(credentials=credentials, project=project_id)
job_config = bigquery.LoadJobConfig(
source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON,
schema=[
bigquery.schema.SchemaField('Ticker', 'STRING', mode='REQUIRED'),
bigquery.schema.SchemaField('Mic', 'STRING', mode='REQUIRED'),
bigquery.schema.SchemaField(
'Contacts',
'RECORD',
mode='REPEATED',
fields=[
bigquery.schema.SchemaField('Name',
'STRING',
mode='NULLABLE'),
bigquery.schema.SchemaField('email',
'STRING',
mode='NULLABLE')
]),
bigquery.schema.SchemaField(
'Dividend',
'RECORD',
mode='REPEATED',
fields=[
bigquery.schema.SchemaField('DeclarationYear',
'STRING',
mode='NULLABLE'),
bigquery.schema.SchemaField('DeclaratioMonth',
'STRING',
mode='NULLABLE'),
bigquery.schema.SchemaField('DeclarationDate',
'STRING',
mode='NULLABLE')
]),
bigquery.schema.SchemaField('RecentDeclarationDate',
'DATE',
mode='NULLABLE'),
bigquery.schema.SchemaField('NextPayableDate',
'DATE',
mode='NULLABLE'),
bigquery.schema.SchemaField('ProbabilityNextMonthDeclaration',
'NUMERIC',
mode='NULLABLE'),
bigquery.schema.SchemaField('Period', 'INTEGER', mode='NULLABLE'),
bigquery.schema.SchemaField('ExpectedStartDate',
'DATE',
mode='NULLABLE'),
bigquery.schema.SchemaField('ExpectedEndDate',
'DATE',
mode='NULLABLE'),
bigquery.schema.SchemaField('LastRunDate', 'DATE', mode='NULLABLE')
],
write_disposition="WRITE_TRUNCATE",
)
s = requests.Session()
retries = Retry(total=5,
backoff_factor=0.1,
status_forcelist=[500, 502, 503, 504, 400],
raise_on_status=True)
s.mount('https://', HTTPAdapter(max_retries=retries))
dataframe = pdgbq.read_gbq(query=query_string, project_id=project_id)
dataframe['NextPayableDate'] = [
d.strftime('%Y-%m-%d') if not pd.isnull(d) else None
for d in dataframe['NextPayableDate']
]
dataframe['ExpectedStartDate'] = [
d.strftime('%Y-%m-%d') if not pd.isnull(d) else None
for d in dataframe['ExpectedStartDate']
]
dataframe['ExpectedEndDate'] = [
d.strftime('%Y-%m-%d') if not pd.isnull(d) else None
for d in dataframe['ExpectedEndDate']
]
dataframe['LastRunDate'] = [
d.strftime('%Y-%m-%d') if not pd.isnull(d) else None
for d in dataframe['LastRunDate']
]
dataframe['RecentDeclarationDate'] = [
d.strftime('%Y-%m-%d') if not pd.isnull(d) else None
for d in dataframe['RecentDeclarationDate']
]
dataframe['Period'] = dataframe['Period'].astype('Int64')
for ind in dataframe.index:
mic = dataframe['Mic'][ind]
if not (mic == 'HSBC_non_local_customer'
or mic == 'HSBC_local_customer'):
declarationDate = dataframe['RecentDeclarationDate'][ind]
params = {
'IdentifierType': 'Symbol',
'Identifier': dataframe.iloc[ind]['Ticker'],
'IdentifierAsOfDate': ' ',
'StartDate': '01/01/2018',
'EndDate': datetime.now().strftime('%m/%d/%Y'),
'CorporateActionsAdjusted': 'True',
'_token': Variable.get("xignitetoken")
}
response = s.get(url=url, params=params)
results = json.loads(response.text)['CashDividends']
if len(results) > 0:
declaredDate = results[0]['DeclaredDate']
if results[0]['DeclaredDate'] == None and results[0][
'ExDate'] != None:
declaredDate = (
datetime.strptime(results[0]['ExDate'], '%Y-%m-%d') -
timedelta(30)).strftime('%Y-%m-%d')
for i in range(0, len(results)):
if not (results[i]['DeclaredDate'] == None
and results[i]['ExDate'] == None):
if results[i]['DeclaredDate'] == None:
dt = datetime.strptime(results[i]['ExDate'],
'%Y-%m-%d') - timedelta(30)
else:
dt = datetime.strptime(results[i]['DeclaredDate'],
'%Y-%m-%d')
year = dt.year
month = dt.month
day = dt.day
if pd.isnull(declarationDate):
baseArray = dataframe.iloc[ind]['Dividend']
if not isinstance(dataframe.iloc[ind]['Dividend'],
(np.ndarray)):
baseArray = []
newDeclaration = {
'DeclarationYear': str(year),
'DeclaratioMonth': str(month),
'DeclarationDate': str(day)
}
appendedDeclaration = np.append(
baseArray, [newDeclaration])
dataframe.at[ind, 'Dividend'] = appendedDeclaration
#query_job.result()
elif dt > datetime.strptime(declarationDate,
'%Y-%m-%d'):
newDeclaration = {
'DeclarationYear': str(year),
'DeclaratioMonth': str(month),
'DeclarationDate': str(day)
}
appendedDeclaration = np.append(
dataframe.iloc[ind]['Dividend'],
[newDeclaration])
dataframe.at[ind, 'Dividend'] = appendedDeclaration
#query_job.result()
if pd.isnull(declarationDate) or (datetime.strptime(
declarationDate, '%Y-%m-%d') < datetime.strptime(
declaredDate, '%Y-%m-%d')):
dataframe.at[ind, 'RecentDeclarationDate'] = declaredDate
dataframe.at[ind,
'NextPayableDate'] = results[0]['PayDate']
#query_job.result()
today = date.today()
today_datetime = datetime(
year=today.year,
month=today.month,
day=today.day,
)
if (pd.isnull(declarationDate) or
(datetime.strptime(declarationDate, '%Y-%m-%d') <
datetime.strptime(declaredDate, '%Y-%m-%d'))) and (
datetime.strptime(results[0]['PayDate'],
'%Y-%m-%d') > today_datetime):
contacts = dataframe["Contacts"][ind]
html_string = None
with open(
'/opt/bitnami/airflow/dags/git-github-com-jainita95-dividend-tracker-git/EmailTemplateDividendDeclared.html',
'r') as f:
html_string = f.read()
html_string = html_string.format(
code=dataframe.iloc[ind]['Ticker'],
date=results[0]['PayDate'])
name = []
emails = []
for i in range(0, contacts.size):
name.append(contacts[i]['Name'])
emails.append(To(contacts[i]['email']))
message = Mail(
from_email='*****@*****.**',
to_emails=emails,
subject="Urgent ! New Dividend Declared for " +
dataframe.iloc[ind]['Ticker'],
html_content=html_string)
with open(
'/opt/bitnami/airflow/dags/git-github-com-jainita95-dividend-tracker-git/hsbcLogo.png',
'rb') as f:
data = f.read()
f.close()
encoded = base64.b64encode(data).decode()
attachment = Attachment()
attachment.file_content = FileContent(encoded)
attachment.file_type = FileType('image/png')
attachment.file_name = FileName('hsbcLogo.png')
attachment.disposition = Disposition('inline')
attachment.content_id = ContentId('hsbclogo')
message.add_attachment(attachment)
try:
sg = SendGridAPIClient(Variable.get("sendgridapikey"))
response = sg.send(message)
except Exception as e:
print(e.message)
json_df = dataframe.to_json(orient="records")
json_data = json.loads(json_df)
job = client.load_table_from_json(
json_data, table_id, job_config=job_config) # Make an API request.
job.result()
def test_parse_retry_after_invalid(self, value):
retry = Retry()
with pytest.raises(InvalidHeader):
retry.parse_retry_after(value)
def test_parse_retry_after(self, value, expected):
retry = Retry()
assert retry.parse_retry_after(value) == expected
"""Service to invoke Rest services."""
import json
import requests
from flask import current_app
from requests.adapters import HTTPAdapter # pylint:disable=ungrouped-imports
# pylint:disable=ungrouped-imports
from requests.exceptions import ConnectionError as ReqConnectionError
from requests.exceptions import ConnectTimeout, HTTPError
from urllib3.util.retry import Retry
from auth_api.exceptions import ServiceUnavailableException
from auth_api.utils.enums import AuthHeaderType, ContentType
RETRY_ADAPTER = HTTPAdapter(
max_retries=Retry(total=5, backoff_factor=1, status_forcelist=[404]))
class RestService:
"""Service to invoke Rest services which uses OAuth 2.0 implementation."""
@staticmethod
def post(endpoint,
token=None,
auth_header_type: AuthHeaderType = AuthHeaderType.BEARER,
content_type: ContentType = ContentType.JSON,
data=None):
"""POST service."""
current_app.logger.debug('<post')
headers = {
'Authorization': auth_header_type.value.format(token),
socket.gethostname(),
})))
tracerProvider = trace.get_tracer_provider()
tracer = tracerProvider.get_tracer(__name__)
tracerProvider.add_span_processor(SimpleSpanProcessor(ConsoleSpanExporter()))
otlp_exporter = OTLPSpanExporter(endpoint="{}:55680".format(OTLP),
insecure=True)
tracerProvider.add_span_processor(SimpleSpanProcessor(otlp_exporter))
FlaskInstrumentor().instrument_app(app)
RequestsInstrumentor().instrument(tracer_provider=tracerProvider)
retry_strategy = Retry(total=2,
status_forcelist=[401, 401.1, 429, 503],
method_whitelist=[
"HEAD", "GET", "POST", "PUT", "DELETE", "OPTIONS",
"TRACE"
])
@app.errorhandler(Error)
def handle_invalid_usage(error):
response = jsonify(error.to_dict())
response.status_code = error.status_code
return response
@app.route("/server_request_login")
def server_request_login():
errorRate = random.randint(0, 99)
with tracer.start_as_current_span("verify_login"):
def test_parse_retry_after_invalid(self, value):
retry = Retry()
with pytest.raises(InvalidHeader):
retry.parse_retry_after(value)
class OmdbApi(ApiBase):
ApiUrl = "https://www.omdbapi.com/"
session = requests.Session()
retries = Retry(
total=5,
backoff_factor=0.1,
status_forcelist=[500, 503, 504, 520, 521, 522, 524],
)
session.mount("https://", HTTPAdapter(max_retries=retries))
def __init__(self):
self.api_key = g.get_setting("omdb.apikey", None)
self.omdb_support = False if not self.api_key else True
self.meta_hash = tools.md5_hash((self.omdb_support, self.ApiUrl))
self.normalization = [
(
"@title",
("title", "originaltitle", "sorttitle"),
lambda d: d if not self._is_value_none(d) else None,
),
("@rated", "mpaa", lambda d: d if not self._is_value_none(d) else None),
(
"@released",
("premiered", "aired"),
lambda d: tools.validate_date(d)
if not self._is_value_none(d)
else None,
),
(
"@runtime",
"duration",
lambda d: int(d[:-4]) * 60 if not self._is_value_none(d) else None,
),
(
"@genre",
"genre",
lambda d: sorted(
OrderedDict.fromkeys({x.strip() for x in d.split(",")})
)
if not self._is_value_none(d)
else None,
),
(
"@director",
"director",
lambda d: sorted(
OrderedDict.fromkeys(
{re.sub(r"\(.*?\)", "", x).strip() for x in d.split(",")}
)
)
if not self._is_value_none(d)
else None,
),
(
"@writer",
"writer",
lambda d: sorted(
OrderedDict.fromkeys(
{re.sub(r"\(.*?\)", "", x).strip() for x in d.split(",")}
)
)
if not self._is_value_none(d)
else None,
),
("@plot", ("plot", "overview", "plotoutline"), None),
(
"@country",
"country",
lambda d: d if not self._is_value_none(d) else None,
),
("@imdbID", ("imdbnumber", "imdb_id"), None),
(
None,
"rating.imdb",
(
("@imdbRating", "@imdbVotes"),
lambda a, c: {
"rating": tools.safe_round(tools.get_clean_number(a), 2),
"votes": tools.get_clean_number(c),
}
if not self._is_value_none(a) and not self._is_value_none(c)
else None,
),
),
(
"@Production",
"studio",
lambda d: d if not self._is_value_none(d) else None,
),
("@awards", "awards", lambda d: d if not self._is_value_none(d) else None),
(
"@awards",
"oscar_wins",
lambda d: self._extract_awards(d, ("Won", "Oscar")),
),
(
"@awards",
"oscar_nominations",
lambda d: self._extract_awards(d, ("Nominated for", "Oscar")),
),
(
"@awards",
"award_wins",
lambda d: self._extract_awards(d, ("Another", "wins"), ("", "wins")),
),
(
"@awards",
"award_nominations",
lambda d: self._extract_awards(
d, ("wins &", "nominations"), ("", "nominations")
),
),
(
"@metascore",
"metacritic_rating",
lambda d: d if not self._is_value_none(d) else None,
),
(
"@tomatoMeter",
"rottentomatoes_rating",
lambda d: d if not self._is_value_none(d) else None,
),
(
"@tomatoImage",
"rottentomatoes_image",
lambda d: d if not self._is_value_none(d) else None,
),
(
"@tomatoReviews",
"rottentomatoes_reviewstotal",
lambda d: tools.get_clean_number(d)
if not self._is_value_none(d)
else None,
),
(
"@tomatoFresh",
"rottentomatoes_reviewsfresh",
lambda d: tools.get_clean_number(d)
if not self._is_value_none(d)
else None,
),
(
"@tomatoRotten",
"rottentomatoes_reviewsrotten",
lambda d: tools.get_clean_number(d)
if not self._is_value_none(d)
else None,
),
(
"@tomatoConsensus",
"rottentomatoes_consensus",
lambda d: d if not self._is_value_none(d) else None,
),
(
"@tomatoUserMeter",
"rottentomatoes_usermeter",
lambda d: d if not self._is_value_none(d) else None,
),
(
"@tomatoUserReviews",
"rottentomatoes_userreviews",
lambda d: tools.get_clean_number(d)
if not self._is_value_none(d)
else None,
),
]
def _extract_awards(self, value, *params):
try:
if self._is_value_none(value):
return None
except AttributeError:
return None
for i in params:
exp = i[0] + "(.+?)" + i[1]
try:
result = re.search(exp, value).group(1).strip()
if not self._is_value_none(result):
return result
except AttributeError:
continue
return None
@staticmethod
def _is_value_none(value):
if value in ["N/A", "0.0", "0", 0, 0.0, None]:
return True
else:
return False
@omdb_guard_response
def get(self, **params):
params.update({"tomatoes": "True"})
params.update({"plot": "full"})
params.update({"r": "xml"})
params.update({"apikey": self.api_key})
return self.session.get(self.ApiUrl, params=params, timeout=10)
@wrap_omdb_object
def get_json(self, **params):
response = self.get(**params)
if response is None:
return None
try:
if not response.content:
return None
return self._handle_response(
xml_to_dict.parse(response.text).get("root", {}).get("movie")
)
except (ValueError, AttributeError):
g.log_stacktrace()
g.log(
"Failed to receive JSON from OMDb response - response: {}".format(
response
),
"error",
)
return None
@handle_single_item_or_list
def _handle_response(self, item):
if item is None:
return None
item = self._try_detect_type(item)
return {"info": self._normalize_info(self.normalization, item)}
@use_cache()
def get_json_cached(self, **params):
return self.get_json(**params)
@staticmethod
@handle_single_item_or_list
def _try_detect_type(item):
if "type" in item:
if "series" == item["type"]:
item.update({"mediatype": "tvshow"})
else:
item.update({"mediatype": item["type"]})
return item
def test_sleep(self):
# sleep a very small amount of time so our code coverage is happy
retry = Retry(backoff_factor=0.0001)
retry = retry.increment()
retry = retry.increment()
retry.sleep()
def test_parse_retry_after(self, value, expected):
retry = Retry()
assert retry.parse_retry_after(value) == expected
def main(shift):
shifttext = str(shift)
with requests.Session() as s:
retries = Retry(total=5,
backoff_factor=1,
method_whitelist=('GET', 'POST'),
status_forcelist=(500, 502, 504))
adapter = HTTPAdapter(max_retries=retries)
s.mount('http://', adapter)
s.mount('https://', adapter)
s.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36 OPR/62.0.3331.116"
try:
print "{0}: Logging in ...".format(shifttext)
login = s.post("https://chopeshift.challenger.sg/login/login_validation", credentials.logindata, timeout=10)
except Exception as exc:
print "{0}: Login failed with error: {1}".format(shifttext, str(exc))
return
print "{0}: Login success".format(shifttext)
cookiestring = ""
for k, v in s.cookies.iteritems():
if not k.startswith("_"):
cookiestring += k + "=" + v +"; "
s.headers["cookie"] = cookiestring[:-1]
try:
print "{0}: Getting calendar ...".format(shifttext)
calendar = s.get("https://chopeshift.challenger.sg/schedule/scheduleCalendar", timeout=10)
except Exception as exc:
print "{0}: Get calendar failed with error: {1}".format(shifttext, str(exc))
logout(s, shifttext)
return
print "{0}: Get calendar success".format(shifttext)
result = ast.literal_eval(calendar.text.encode("UTF-8").replace("null", "None").strip())
calendar_dict = {}
accepted_dict = {}
for r in result:
slotno = str(r["slotno"]).strip()
if not slotno.startswith("RA"):
continue
statusslot = str(r["statusslot"]).strip().lower()
date = "{0}/{1}/{2}".format(str(r["day"].zfill(2)), str(r["month"].zfill(2)), str(r["year"]))
time = str(r["time"]).strip().upper()
location = str(r["location"]).strip().upper()
if statusslot == "open":
calendar_dict[slotno] = [date, time, location]
elif statusslot == "accepted":
accepted_dict.setdefault(date, []).append(time)
if shift[0] in accepted_dict:
if shift[1] in accepted_dict[shift[0]]:
print "{0}: Already have a shift at this time slot".format(shifttext)
logout(s, shifttext)
return
print "{0}: Try booking ...".format(shifttext)
found = False
for k, v in calendar_dict.iteritems():
if v == shift:
try:
chope = s.post("https://chopeshift.challenger.sg/schedule/book", {"slotNo": k, "extend": None}, timeout=10)
print "{0}: Shift booked successfully with status code {1} ({2})".format(shifttext, str(chope.status_code), str(chope.text))
found = True
break
except Exception as exc:
print "{0}: Shift booked failed with error: {1}".format(str(exc))
if found == False:
print "{0}: No matching shift found".format(shifttext)
logout(s, shifttext)
return