def _set_info(self):
""""""
if self.token is None:
return None
result = requests.get(url=self.__users_url,
data=None,
headers=self.headers)
dict_result = json.loads(result.content)
for i in range(len(dict_result['users'])):
if dict_result['users'][i]['name'] == self.name:
self.identification = dict_result['users'][i]['id']
break
elif i == len(dict_result['users']) - 1:
"""Record error info"""
self.identification = None
warnings.warn('No this user!', Warning)
else:
continue
result = requests.get(url=self.__users_url + '/' + self.identification,
data=None,
headers=self.headers)
dict_result = json.loads(result.content)
self.enable = dict_result['user']['enabled']
self.endpoint["keystone"] = utils.get_endpoint(self.token, "keystone")
self.endpoint["nova"] = utils.get_endpoint(self.token, "nova")
self.endpoint["cinder"] = utils.get_endpoint(self.token, "cinder")
self.endpoint["glance"] = utils.get_endpoint(self.token, "glance")
self.endpoint["neutron"] = utils.get_endpoint(self.token, "neutron")
def image_content_batch(token, urls, categories=None, threshold=None):
endpoint = utils.get_endpoint(ais.AisService.MODERATION_SERVICE)
_url = 'https://%s/v1.0/moderation/image/batch' % endpoint
_data = {
"urls": urls,
"categories": categories,
"threshold": threshold,
}
kreq = urllib2.Request(url=_url)
kreq.add_header('Content-Type', 'application/json')
kreq.add_header('X-Auth-Token', token)
kreq.add_data(json.dumps(_data))
resp = None
status_code = None
try:
#
# Here we use the unvertified-ssl-context, Because in FunctionStage
# the client CA-validation have some problem, so we must do this.
#
_context = ssl._create_unverified_context()
r = urllib2.urlopen(kreq, context=_context)
#
# We use HTTPError and URLError，because urllib2 can't process the 4XX &
# 500 error in the single urlopen function.
#
# If you use a modern, high-level designed HTTP client lib, Yeah, I mean requests,
# there is no this problem.
#
except HTTPError, e:
resp = e.read()
status_code = e.code
def dark_enhance(token, image, brightness=0.9):
endpoint = utils.get_endpoint(ais.AisService.IMAGE_SERVICE)
_url = 'https://%s/v1.0/vision/dark-enhance' % endpoint
_data = {"image": image, "brightness": brightness}
kreq = urllib2.Request(url=_url)
kreq.add_header('Content-Type', 'application/json')
kreq.add_header('X-Auth-Token', token)
kreq.add_data(json.dumps(_data))
resp = None
status_code = None
try:
#
# Here we use the unvertified-ssl-context, Because in FunctionStage
# the client CA-validation have some problem, so we must do this.
#
_context = ssl._create_unverified_context()
r = urllib2.urlopen(kreq, context=_context)
#
# We use HTTPError and URLError，because urllib2 can't process the 4XX &
# 500 error in the single urlopen function.
#
# If you use a modern, high-level designed HTTP client lib, Yeah, I mean requests,
# there is no this problem.
#
except HTTPError, e:
resp = e.read()
status_code = e.code
def image_batch_jobs(token, urls, categories=['politics', 'terrorism', 'p**n']):
endpoint = utils.get_endpoint(ais.AisService.MODERATION_SERVICE)
status, r = _image_batch_jobs(endpoint, token, urls, categories)
if status != 200:
return r
submit_result = json.loads(r)
job_id = submit_result['result'].get('job_id', '')
# print "Process job id is :", job_id
time.sleep(1.0)
try:
while True:
status, r = _get_result(endpoint, token, job_id)
if status != 200:
return r
rec_result = json.loads(r)
process_status = rec_result["result"].get('status')
if process_status == 'failed':
return r
elif process_status == 'finish':
return r
#
# process_status == 0 || process_status == 1
#
else:
time.sleep(2.0)
continue
except Exception:
return ''
def __init__(self):
self.token = utils.get_token()
self.enable = True
self.__headers = {
'User-Agent': utils.init_data().get('agent', 'ALL'),
'X-Auth-Token': self.token,
"Content-type": "application/json",
"Accept": "application/json"
}
self.__roles_url = utils.get_endpoint(self.token,
'keystone') + '/roles'
def image_tagging_aksk(_ak,
_sk,
image,
url,
languzge,
limit=-1,
threshold=0.0):
endpoint = utils.get_endpoint(ais.AisService.IMAGE_SERVICE)
_url = 'https://%s/v1.0/image/tagging' % endpoint
sig = signer.Signer()
sig.AppKey = _ak
sig.AppSecret = _sk
_data = {
"image": image,
"url": url,
"language": languzge,
"limit": limit,
"threshold": threshold
}
kreq = signer.HttpRequest()
kreq.scheme = "https"
kreq.host = endpoint
kreq.uri = "/v1.0/image/tagging"
kreq.method = "POST"
kreq.headers = {"Content-Type": "application/json"}
kreq.body = json.dumps(_data)
resp = None
status_code = None
try:
sig.Sign(kreq)
#
# Here we use the unvertified-ssl-context, Because in FunctionStage
# the client CA-validation have some problem, so we must do this.
#
_context = ssl._create_unverified_context()
req = urllib2.Request(url=_url, data=kreq.body, headers=kreq.headers)
r = urllib2.urlopen(req, context=_context)
#
# We use HTTPError and URLError，because urllib2 can't process the 4XX &
# 500 error in the single urlopen function.
#
# If you use a modern, high-level designed HTTP client lib, Yeah, I mean requests,
# there is no this problem.
#
except HTTPError, e:
resp = e.read()
status_code = e.code
def moderation_text_aksk(
_ak,
_sk,
text,
type='content',
categories=["ad", "politics", "p**n", "abuse", "contraband", "flood"]):
endpoint = utils.get_endpoint(ais.AisService.MODERATION_SERVICE)
_url = 'https://%s/v1.0/moderation/text' % endpoint
sig = signer.Signer()
sig.AppKey = _ak
sig.AppSecret = _sk
_data = {
"categories":
categories, # 检测场景 Array politics：涉政 p**n：涉黄 ad：广告 abuse：辱骂 contraband：违禁品 flood：灌水
"items": [{
"text": text,
"type": type
} # items: 待检测的文本列表 text 待检测文本 type 文本类型
]
}
kreq = signer.HttpRequest()
kreq.scheme = "https"
kreq.host = endpoint
kreq.uri = "/v1.0/moderation/text"
kreq.method = "POST"
kreq.headers = {"Content-Type": "application/json"}
kreq.body = json.dumps(_data)
resp = None
status_code = None
try:
sig.Sign(kreq)
#
# Here we use the unvertified-ssl-context, Because in FunctionStage
# the client CA-validation have some problem, so we must do this.
#
_context = ssl._create_unverified_context()
req = urllib2.Request(url=_url, data=kreq.body, headers=kreq.headers)
r = urllib2.urlopen(req, context=_context)
#
# We use HTTPError and URLError，because urllib2 can't process the 4XX &
# 500 error in the single urlopen function.
#
# If you use a modern, high-level designed HTTP client lib, Yeah, I mean requests,
# there is no this problem.
#
except HTTPError, e:
resp = e.read()
status_code = e.code
def moderation_video_aksk(_ak,
_sk,
url,
frame_interval=5,
categories=['politics', 'terrorism']):
sig = signer.Signer()
sig.AppKey = _ak
sig.AppSecret = _sk
endpoint = utils.get_endpoint(ais.AisService.MODERATION_SERVICE)
status, r = _moderation_video_aksk(endpoint, sig, url, frame_interval,
categories)
if status != 200:
return r
submit_result = json.loads(r)
job_id = submit_result['result'].get('job_id', '')
#print "Process job id is :", job_id
time.sleep(1.0)
try:
while True:
status, r = _get_result_aksk(endpoint, sig, job_id)
if status != 200:
return r
rec_result = json.loads(r)
process_status = rec_result["result"].get('status')
if process_status == 'failed':
return r
elif process_status == 'finish':
return r
#
# process_status == 0 || process_status == 1
#
else:
time.sleep(2.0)
continue
except Exception:
return ''
def moderation_text(
token,
text,
type='content',
categories=["ad", "politics", "p**n", "abuse", "contraband", "flood"]):
endpoint = utils.get_endpoint(ais.AisService.MODERATION_SERVICE)
_url = 'https://%s/v1.0/moderation/text' % endpoint
_data = {
"categories":
categories, # 检测场景 Array politics：涉政 p**n：涉黄 ad：广告 abuse：辱骂 contraband：违禁品 flood：灌水
"items": [{
"text": text,
"type": type
} # items: 待检测的文本列表 text 待检测文本 type 文本类型
]
}
kreq = urllib2.Request(url=_url)
kreq.add_header('Content-Type', 'application/json')
kreq.add_header('X-Auth-Token', token)
kreq.add_data(json.dumps(_data))
resp = None
status_code = None
try:
#
# Here we use the unvertified-ssl-context, Because in FunctionStage
# the client CA-validation have some problem, so we must do this.
#
_context = ssl._create_unverified_context()
r = urllib2.urlopen(kreq, context=_context)
#
# We use HTTPError and URLError，because urllib2 can't process the 4XX &
# 500 error in the single urlopen function.
#
# If you use a modern, high-level designed HTTP client lib, Yeah, I mean requests,
# there is no this problem.
#
except HTTPError, e:
resp = e.read()
status_code = e.code
def create(self, project, name, passwd, role, **kwargs):
params = {
"user": {
"name": name,
"password": passwd,
"default_project_id": project.identification,
"enabled": True,
}
}
# TODO(dc):description is not concerned
# if kwargs.get("description"):
# params["user"]["description"] = kwargs.get("description")
if kwargs.get("domain_id"):
params["user"]["domain_id"] = kwargs.get("domain_id")
user_id = json.loads(
requests.post(url=self.__users_url,
data=json.dumps(params),
headers=self.__headers).content)["user"]["id"]
# TODO(dc): domain is not concerned
requests.put(url=utils.get_endpoint(self.token, 'keystone') +
'/projects/' + project.identification + '/users/' +
user_id + '/roles/' + role.identification,
headers=self.__headers)
return self.get(name=name, passwd=passwd, project=project)
import os.path
import re
import sys
import time
import traceback
import urllib2
from datetime import datetime
import requests
from bs4 import BeautifulSoup
from lxml import etree
from lxml.etree import tostring
from utils import send_to_slack, notify_users_about_article, get_endpoint, is_production, write_news
ENDPOINT = get_endpoint()
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
logging.getLogger("requests").setLevel(logging.WARNING)
if os.path.isfile("config.ini"):
config = ConfigParser.RawConfigParser(allow_no_value=True)
config.read('config.ini')
APPSPOT_KEY = config.get("appspot", "X-Secret-Key")
else:
print "config.ini is needed"
sys.exit(1)
class NewsParser:
TIMEOUT = 15
RE_POST_ID = re.compile(ur'((?<=")post-\d+(?="))')
def feeler_coords(self, angle, length):
'''Coordinates of feeler with given angle and length.'''
end_x, end_y = utils.get_endpoint(self.critter.coords[0], self.critter.coords[1],
(self.critter.heading + angle) % 360, length)
return self.critter.coords[0], self.critter.coords[1], end_x, end_y
def mouth_end(self):
'''Coordinates of the point where the mouth opens.'''
return utils.get_endpoint(self.coords[0], self.coords[1], self.heading, Thing.radius)
#!/usr/bin/python
# -*- coding: iso-8859-15 -*-
import codecs
import ConfigParser
import json
import os
import re
import requests
import urllib2
from utils import get_endpoint, notify_users, read_json, write_json, write_news
ENDPOINT = get_endpoint()
if os.path.isfile("config.ini"):
config = ConfigParser.RawConfigParser(allow_no_value=True)
config.read('config.ini')
APPSPOT_KEY = config.get("appspot", "X-Secret-Key")
class BuliParser:
def __init__(self,
season,
league,
relay,
push_descr,
fragment_id,
leage_relay="LEAGUE_RELAY"):
self.league = league
self.relay = relay
self.iat_season_base = "https://www.iat.uni-leipzig.de/datenbanken/blgew{0}/".format(
season)
import os
import argparse
import traceback
from parser.buli_parser import BuliParser
from parser.news_parser import BVDGParser, SpeyerParser, SchwedtParser, MutterstadtParser, RodingParser
from utils import update_readme, commit_changes, send_to_slack, get_endpoint, is_production, NEWS_FILE
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description="Send new articles or competition results to the server")
parser.add_argument('--notify', action='store_true')
args = parser.parse_args()
print "Endpoint: " + get_endpoint()
SEASON = "1718"
BuliParser1A = BuliParser(SEASON,
"1",
"Gruppe+A",
"1. Bundesliga - Staffel A",
"4",
leage_relay="1a")
BuliParser1B = BuliParser(SEASON,
"1",
"Gruppe+B",
"1. Bundesliga - Staffel B",
"5",
leage_relay="1b")
BuliParser2A = BuliParser(SEASON,
"2",
"Gruppe+A",
"2. Bundesliga - Staffel A",
def get_values(self, points, calculationtype=None, starttime='*-1d',
endtime='*', boundary=None, boundarytype=None,
maxcount='1000', desiredunits=None, interval=None,
intervals='24', retrievalmode='Auto', summarytype='Total',
calculationbasis='TimeWeighted', timetype='Auto',
summaryduration=None, sampletype='ExpressionRecordedValues',
sampleinterval=None, time=None, filterexpression=None,
includefilteredvalues=False, sortorder='Ascending',
append=False, overwrite=False):
# starttime is Time String
# endtime is Time String
# interval is AFTimeSpan
# desiredUnits is a uom, cannot be specified for PI points
# fiterexpression is filtering like * or SINU*
# includefilteredvalues bool: Specify 'true' to indicate that values which fail the filter criteria are present in the returned data at the times where they occurred with a value set to a 'Filtered' enumeration value with bad status. Repeated consecutive failures are omitted.
# sortorder default is 'Ascending'
# summaryDuration The duration of each summary interval. If specified in hours, minutes, seconds, or milliseconds, the summary durations will be evenly spaced UTC time intervals. Longer interval types are interpreted using wall clock rules and are time zone dependent.
# TODO: add starttime parameter
# TODO: add endtime parameter
# TODO: add boundary parameter
# TODO: add interval parameter
calctype = calculationtype.lower()
is_single_value = True if calctype == 'current' or calctype == 'end' \
else False
log.debug('Calculation type: %s, Single value: %s', calctype,
is_single_value)
for point in iterfy(points):
log.debug('Retrieving %s data for %s...', calctype, point.name)
if calctype == 'current':
payload = {'time': time}
elif calctype == 'interpolated':
payload = {'startTime': starttime, 'endTime': endtime,
'interval': interval,
'filterExpression': filterexpression,
'includeFilteredValues': includefilteredvalues}
elif calctype == 'interpolatedattimes':
payload = {'time': time, 'filterExpression': filterexpression,
'includeFilteredValues': includefilteredvalues,
'sortOrder': sortorder}
elif calctype == 'recorded':
payload = {'startTime': starttime, 'endTime': endtime,
'boundaryType': boundarytype,
'filterExpression': filterexpression,
'includeFilteredValues': includefilteredvalues,
'maxCount': maxcount}
elif calctype == 'recordedattime':
payload = {'time': time, 'retrievalMode': retrievalmode}
elif calctype == 'plot':
payload = {'startTime': starttime, 'endTime': endtime,
'intervals': intervals}
elif calctype == 'summary':
payload = {'startTime': starttime, 'endTime': endtime,
'summaryType': summarytype,
'calculationBasis': calculationbasis,
'timeType': timetype,
'summaryDuration': summaryduration,
'sampleType': sampletype,
'sampleInterval': sampleinterval,
'filterExpression': filterexpression}
elif calctype == 'end':
payload = {}
else:
log.debug('This %s request has no URL parameters', calctype)
endpoint = get_endpoint(self.url, point, calctype)
# TODO: add queryParamater generator function here?
try:
log.debug('Instantiating %s request for PI point %s to '
'endpoint %s with the following parameters: %s',
calctype, point.name, endpoint, payload)
r = self.session.get(endpoint, params=payload)
if r.status_code != requests.codes.ok:
r.raise_for_status()
except OSIsoftPyException as e:
log.error('Exception while retrieving recorded values'
'from %s for %s. Raw JSON: %s', endpoint, point.name,
exc_info=True)
data = r.json()
log.debug('HTTP %s - Instantiating OSIsoftPy.Values()',
r.status_code)
log.debug('Staging PI point value for '
'instantiation...')
try:
new_values = get_point_values(point, calctype, data)
log.debug('%s %s value(s) were instantiated for %s.',
get_count(new_values), calctype, point.name)
except OSIsoftPyException as e:
log.error('Exception while instantiating PI Point value(s)'
'for %s. Raw JSON: %s', point.name, data,
exc_info=True)
current_values = TypedList(validtypes=Value)
if is_single_value:
try:
value = getattr(point, get_attribute(calctype))
log.debug('Storing %s value.', calctype)
current_values.append(value)
except TypeError as e:
log.warning('TypeError encountered - the attribute %s is '
'empty for %s, which will raise an '
'exception when trying to iterate.',
get_attribute(calctype), point.name,
exc_info=False)
else:
try:
for value in getattr(point, get_attribute(calctype)):
log.debug(
'Storing %s value for PI point %s, attribute: %s',
calctype, point.name, get_attribute(calctype))
current_values.append(value)
except TypeError as e:
log.warning('TypeError encountered - the attribute %s is '
'empty for %s, which will raise an '
'exception when trying to iterate.',
get_attribute(calctype), point.name,
exc_info=False)
log.debug('PI point %s currently has %s %s values.', point.name,
get_count(current_values), calctype)
if is_single_value and overwrite:
log.debug('Single point value - overwriting existing %s '
'value, Single value: %s.', calctype,
is_single_value)
setattr(point, get_attribute(calctype), new_values[0])
elif is_single_value and append:
log.debug('Single point value - append is true but cannot '
'append...overwriting existing %s '
'value, Single value: %s.', calctype,
is_single_value)
setattr(point, get_attribute(calctype), new_values[0])
elif not is_single_value and overwrite:
log.debug('Multiple point values - overwriting %s existing '
'%s values, Single value: %s.',
get_count(current_values), calctype, is_single_value)
setattr(point, get_attribute(calctype), new_values)
elif not is_single_value and append:
for new_value in new_values:
current_values.append(new_value)
setattr(point, get_attribute(calctype), current_values)
else:
# TODO: allow both to be false if no data exists.
log.error('Error saving %s new %s point value(s) for PI '
'point %s. Single value: %s, Overwrite: %s, Append: '
'%s.', get_count(new_values), calctype, point.name,
is_single_value, overwrite, append)
return points
import os
import argparse
import traceback
from parser.buli_parser import BuliParser
from parser.news_parser import BVDGParser, SpeyerParser, SchwedtParser, MutterstadtParser, RodingParser
from utils import update_readme, commit_changes, send_to_slack, get_endpoint, is_production, NEWS_FILE
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Send new articles or competition results to the server")
parser.add_argument('--notify', action='store_true')
args = parser.parse_args()
print "Endpoint: " + get_endpoint()
SEASON = "1718"
BuliParser1A = BuliParser(SEASON, "1", "Gruppe+A", "1. Bundesliga - Staffel A", "4", leage_relay="1a")
BuliParser1B = BuliParser(SEASON, "1", "Gruppe+B", "1. Bundesliga - Staffel B", "5", leage_relay="1b")
BuliParser2A = BuliParser(SEASON, "2", "Gruppe+A", "2. Bundesliga - Staffel A", "7", leage_relay="2a")
BuliParser2B = BuliParser(SEASON, "2", "Gruppe+B", "2. Bundesliga - Staffel B", "8", leage_relay="2b")
blog_parsers_instances = [BVDGParser(), SpeyerParser(), SchwedtParser(), MutterstadtParser(), RodingParser()]
try:
for parser in [BuliParser1A, BuliParser1B, BuliParser2A, BuliParser2B]:
parser.update_buli(args.notify)
for blog_parser_instance in blog_parsers_instances:
blog_parser_instance.parse_articles(args.notify)
update_readme(blog_parsers_instances)
if is_production():
