async def make_async_request(session, request):
logger.debug('operation=request_started, request=%r', request)
method = request.method
with async_timeout.timeout(request.timeout):
session_method = getattr(session, method.lower())
async with session_method(request.url,
params=request.params,
data=request.body,
headers=request.headers,
**request.kwargs) as client_response:
content_type = client_response.headers.get('Content-Type', '')
if 'text' in content_type:
body = await client_response.text()
elif 'json' in content_type:
body = await client_response.text()
if body:
body = json.loads(body)
else:
body = await client_response.read()
response = Response(url=str(client_response.url),
method=method,
body=body,
headers=dict(client_response.headers),
status_code=client_response.status,
client_response=client_response)
logger.debug('operation=request_finished, request=%r, response=%r',
request, response)
return response
def make_request(session, request):
logger.debug('operation=request_started, request=%r', request)
method = request.method
session_method = getattr(session, method.lower())
client_response = session_method(request.url,
params=request.params,
data=request.body,
headers=request.headers,
timeout=request.timeout,
**request.kwargs)
content_type = client_response.headers.get('Content-Type', '')
if 'text' in content_type:
body = client_response.text
elif 'json' in content_type:
body = client_response.text
if body:
body = json.loads(body)
else:
body = client_response.content
response = Response(url=client_response.url,
method=method,
body=body,
headers=client_response.headers,
status_code=client_response.status_code,
client_response=client_response)
logger.debug('operation=request_finished, request=%r, response=%r',
request, response)
return response
def extra_data_loads(cls, json_string, validate=True):
form = cls.extra_form_class(data=json.loads(json_string or '{}'))
validate_form(form) if validate else form.full_clean()
return form_data(form)
config = {
'user': '******',
'password': '******',
'host': 'db-dev.seldatdirect.com',
'database': 'pldm_dev',
'raise_on_warnings': True,
'use_pure': False,
}
cnx = mysql.connector.connect(**config)
cursor = cnx.cursor()
url = 'http://api.timezonedb.com/v2/list-time-zone?key=BWYLPCYMUJIZ&format=json'
response = requests.get(url)
data = json.loads(response.text)
zones = data['zones']
for zone in zones:
utc = zone['gmtOffset'] / 60 / 60
time = zone['zoneName'].replace("/", ' - ')
du = zone['gmtOffset'] % 3600
if utc > 0:
utc = '+' + str(utc).zfill(2)
else:
utc = abs(utc)
utc = '-' + str(utc).zfill(2)
timeName = '(' + 'UTC ' + utc + ":" + str(du / 60).zfill(2) + ') ' + time
add_employee = ("INSERT INTO timezones (cd, name) VALUES (%s, %s)")
data_employee = (zone['zoneName'], timeName)
cursor.execute(add_employee, data_employee)
con = sqlite3.connect('Wiki_Edits_24h.db')
cur = con.cursor()
cur.execute('DROP TABLE IF EXISTS edits')
cur.execute('''CREATE TABLE IF NOT EXISTS edits(''' + string_sql + ''')''')
## Initialisations for acquisiton loop
## While Loop condition to compare time passed with target time
t_start = time.time()
t_diff = time.time() - t_start
iteration = 0
while t_diff < (hrs_to_log * 60 * 60):
## Get latest wiki edit from websocket and create python dictionary
result = ws.recv()
result_dict = json.loads(result)
## Add time and index
result_dict['time'] = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y/%m/%d %H:%M:%S')
result_dict['id'] = iteration
iteration = iteration + 1
## Transform bools into ints
result_dict['is_anon'] = int(result_dict['is_anon'])
result_dict['is_bot'] = int(result_dict['is_bot'])
result_dict['is_minor'] = int(result_dict['is_minor'])
result_dict['is_new'] = int(result_dict['is_new'])
result_dict['is_unpatrolled'] = int(result_dict['is_unpatrolled'])
## Get geo data from dict and set country, lat and lon as keys in main dict
def extra_data_loads(cls, json_string, validate=True):
form = cls.extra_form_class(data=json.loads(json_string or '{}'))
validate_form(form) if validate else form.full_clean()
return form_data(form)