def load(self):
"""
Loads a config.json file and run a validation process.
If the configurations seem valid, returns Configuration object.
"""
util.set_working_directory()
# paths relatively to this script's location
schema_json = util.json_decode(self.schema_path)
if schema_json is None:
msg = "Problem has occurred during the decoding procedure" + \
" with the following file: " + self.schema_path + "."
logging.error(msg)
raise IOError(msg)
tools_json = util.json_decode(self.tools_path)
if tools_json is None:
msg = "Problem has occurred during the decoding procedure" + \
" with the following file: " + self.tools_path + "."
logging.error(msg)
raise IOError(msg)
try:
with open(self.config_path, mode="r") as file:
config_string = file.read()
decoder = json.JSONDecoder(object_pairs_hook=checking_hook)
config_json = decoder.decode(config_string)
except IOError:
msg = "The file does not exist or cannot be read:" + \
(os.path.split(self.config_path))[1]
logging.error(msg)
raise IOError(msg)
except ValueError as value_error:
msg = (os.path.split(self.config_path))[1] + " file is not valid"
logging.error(msg)
print(value_error)
raise ValidationError(msg)
except KeyError as k_error:
msg = "Duplicate key specified."
logging.error(msg)
print(k_error)
print("Modify: " + (os.path.split(self.config_path))[1])
raise ValidationError(msg)
valid = validation.is_valid_json(config_json, schema_json)
if not valid:
msg = "Validation failed for " + \
(os.path.split(self.config_path))[1] + "."
logging.error(msg)
raise ValidationError(msg)
config = Configuration()
config.iterations = config_json["IterationCount"]
config.runs = config_json["Runs"]
config.queries = config_json["Queries"]
config.scenarios = config_json["Scenarios"]
config.sizes = util.get_power_of_two(config_json["MinSize"], config_json["MaxSize"])
config.tools = config_json["Tools"]
config.optional_arguments = config_json["OptionalArguments"]
return config
def read_config(args):
if not os.path.isfile("config.json"):
print "config.json does not exist! Please copy config-sample.json to config.json and edit to your liking, then run the script."
sys.exit(1)
countries = args.country
if isinstance(countries, basestring):
countries = [countries]
countries = [country.lower().strip() for country in countries]
for country in countries:
if not os.path.isfile("proxies/proxies-%s.json" % country):
print "The proxy configuration file proxies-%s.json does not exist! Exiting." % country
sys.exit(1)
content = util.get_contents("config.json")
config = util.json_decode(content)
if args.ip:
config["public_ip"] = args.ip
if args.bind_ip:
config["bind_ip"] = args.ip
if args.base_ip:
config["base_ip"] = args.base_ip
if args.base_port:
config["base_port"] = args.base_port
if not config["public_ip"]:
try:
print("Autodetecting public IP address...")
public_ip = urllib2.urlopen("http://l2.io/ip").read().strip()
print("Detected public IP as %s. If it's wrong, please cancel the script now and set it in config.json or specify with --ip" % public_ip)
time.sleep(1)
config["public_ip"] = public_ip
except:
print("Could not detect public IP. Please update the public_ip setting in config.json or specify with --ip.")
sys.exit(1)
if args.save:
util.put_contents('config.json', util.json_encode(config))
groups = {}
for country in countries:
groups.update(util.json_decode(util.get_contents("proxies/proxies-%s.json" % country)))
if args.only:
only = set(args.only)
for item in args.only:
if item not in groups:
print "Nonexistent Item: %s, exiting" % item
sys.exit()
for item in groups.keys():
if item not in only:
del groups[item]
elif args.skip:
for item in args.skip:
del groups[item]
config["groups"] = groups
return config
def get_ssh_state():
"""Getting the state of the remote queue from a text file"""
try:
filename = os.path.expanduser('~/glidein_state')
return json_decode(open(filename).read())
except Exception:
logger.warn('error getting ssh state', exc_info=True)
def get_ssh_state():
"""Getting the state of the remote queue from a text file"""
try:
filename = os.path.expanduser('~/glidein_state')
return json_decode(open(filename).read())
except Exception:
logger.warn('error getting ssh state', exc_info=True)
def put(self, classname, objectId):
"""Update a object
"""
self.classname = classname
data = json_decode(self.request.body)
self.objectid = ObjectId(objectId)
result = self.db[self.collection].update({"_id": self.objectid}, data)
def get(self, dir, seq):
opts = {
'startkey': [seq, 'z'],
'endkey': [seq],
'descending': True,
'limit': 1,
}
cur = self.db.view('seq/%s' % dir, **opts)
msg = cur.rows[0].value if cur.rows else None
return msg if msg is None else util.json_decode(msg)
def get(self, dir, seq):
opts = {
'startkey': [seq, 'z'],
'endkey': [seq],
'descending': True,
'limit': 1,
}
cur = self.db.view('seq/%s' % dir, **opts)
msg = cur.rows[0].value if cur.rows else None
return msg if msg is None else util.json_decode(msg)
def __init__(self, **kwargs):
self.project_id = kwargs["project_id"]
self.jsonkey = kwargs["jsonkey"]
self.appname = kwargs["appname"]
self.instanceid = kwargs["instanceid"]
jsonData = json_decode(self.jsonkey)
self.oauth_client = ServiceAccountCredentials.from_json_keyfile_dict(
jsonData, SCOPES
)
self.endpoint = "%s/v1/projects/%s/messages:send" % (BASE_URL, self.project_id)
def post(self, classname):
"""Create collections
"""
self.classname = classname
try:
data = json_decode(self.request.body)
except Exception as ex:
self.send_response(BAD_REQUEST, ex)
self.add_to_log("Add object to %s" % self.classname, data)
objectId = self.db[self.collection].insert(data)
self.send_response(OK, dict(objectId=objectId))
def get_format(self, tool):
"""
Returns the tool's format as string.
"""
# path relatively to this script's location
current_directory = os.getcwd()
util.set_working_directory()
formats_json = util.json_decode(self.dependencies_path)
if formats_json is None:
raise IOError("Problem has occurred during the decoding procedure" +
" with the following file: " + self.dependencies_path)
util.set_working_directory(current_directory)
if tool not in formats_json:
raise IOError("Format for " + tool + " not specified.")
else:
return formats_json[tool]["format"]
def post(self):
"""Create a new token
"""
if not self.can("create_token"):
self.send_response(FORBIDDEN,
dict(error="No permission to create token"))
return
logging.info(self.request.body)
data = json_decode(self.request.body)
device = data.get("device", DEVICE_TYPE_IOS).lower()
channel = data.get("channel", "default")
devicetoken = data.get("token", "")
if device == DEVICE_TYPE_IOS:
if len(devicetoken) != 64:
self.send_response(BAD_REQUEST, dict(error="Invalid token"))
return
try:
binascii.unhexlify(devicetoken)
except Exception as ex:
self.send_response(BAD_REQUEST, dict(error="Invalid token"))
token = EntityBuilder.build_token(devicetoken, device, self.appname,
channel)
try:
result = self.db.tokens.update(
{
"device": device,
"token": devicetoken,
"appname": self.appname
},
token,
upsert=True,
)
# result
# {u'updatedExisting': True, u'connectionId': 47, u'ok': 1.0, u'err': None, u'n': 1}
if result["updatedExisting"]:
self.add_to_log("Token exists", devicetoken)
self.send_response(OK)
else:
self.send_response(OK)
self.add_to_log("Add token", devicetoken)
except Exception as ex:
self.add_to_log("Cannot add token", devicetoken, "warning")
self.send_response(INTERNAL_SERVER_ERROR, dict(error=str(ex)))
def request(self, methodname, kwargs):
"""Send request to RPC Server"""
# check method name for bad characters
if methodname[0] == '_':
logger.warning('cannot use RPC for private methods')
raise Exception('Cannot use RPC for private methods')
# translate request to json
body = json_encode({
'jsonrpc': '2.0',
'method': methodname,
'params': kwargs,
'id': Client.newid()
})
headers = {'Content-type': 'application/json'}
request = urllib2.Request(self._address, data=body, headers=headers)
# make request to server
try:
response = urllib2.urlopen(request, timeout=self._timeout)
except Exception:
logger.warn('error making jsonrpc request', exc_info=True)
raise
# translate response from json
try:
cb_data = response.read()
data = json_decode(cb_data)
except Exception:
try:
logger.info('json data: %r', cb_data)
except Exception:
pass
raise
if 'error' in data:
try:
raise Exception('Error %r: %r %r' % data['error'])
except Exception:
raise Exception('Error %r' % data['error'])
if 'result' in data:
return data['result']
else:
return None
def get(self):
"""Query users
"""
where = self.get_argument("where", None)
if not where:
data = {}
else:
try:
# unpack query conditions
data = json_decode(where)
except Exception as ex:
self.send_response(BAD_REQUEST, dict(error=str(ex)))
cursor = self.db.users.find(data)
users = []
for u in cursor:
users.append(u)
self.send_response(OK, users)
def get(self, classname):
"""Query collection
"""
self.classname = classname
where = self.get_argument("where", None)
if not where:
data = {}
else:
try:
# unpack query conditions
data = json_decode(where)
except Exception as ex:
self.send_response(BAD_REQUEST, dict(error=str(ex)))
objects = self.db[self.collection].find(data)
results = []
for obj in objects:
results.append(obj)
self.send_response(OK, results)
async def test_fcm(self, req, oauth):
self.maxDiff = None
kwargs = {
"project_id": "xxx",
"jsonkey": "{}",
"appname": "",
"instanceid": ""
}
fcm = FCMClient(**kwargs)
accesstoken, e = fcm.oauth_client.get_access_token()
self.assertEqual(accesstoken, "access_token")
response = await fcm.process(
token="aaa",
alert="alert",
fcm={
"android": {
"droid": 1
},
"apns": {
"apple": 2
},
"webpush": {},
"data": {},
},
)
self.assertDictEqual(
json_decode(response),
{
"message": {
"token": "aaa",
"notification": {
"body": "alert",
"title": "alert"
},
"android": {
"droid": 1
},
"apns": {
"apple": 2
},
}
},
)
async def post(self):
if not self.can("send_broadcast"):
self.send_response(FORBIDDEN,
dict(error="No permission to send broadcast"))
return
# if request body is json entity
data = json_decode(self.request.body)
# the channel to be broadcasted
channel = data.get("channel", "default")
# device type
device = data.get("device", None)
# iOS and Android shared param
if type(data["alert"]) is not dict:
alert = "".join(data.get("alert", "").splitlines())
else:
alert = data["alert"]
# iOS
sound = data.get("sound", None)
badge = data.get("badge", None)
if type(data["alert"]) is not dict:
self.add_to_log("%s broadcast" % self.appname, alert, "important")
else:
self.add_to_log(
"%s broadcast" % self.appname,
alert["title"] + ": " + alert["body"],
"important",
)
await self.application.send_broadcast(
self.appname,
self.db,
channel=channel,
alert=alert,
sound=sound,
badge=badge,
device=device,
fcm=data.get("fcm", {}),
wns=data.get("wns", {}),
apns=data.get("apns", {}),
)
delta_t = time.time() - self._time_start
logging.info("Broadcast took time: %sms" % (delta_t * 1000))
self.send_response(ACCEPTED)
def request(self, methodname, kwargs):
"""Send request to RPC Server"""
# check method name for bad characters
if methodname[0] == '_':
logger.warning('cannot use RPC for private methods')
raise Exception('Cannot use RPC for private methods')
# translate request to json
body = json_encode({'jsonrpc': '2.0', 'method': methodname,
'params': kwargs, 'id': Client.newid()})
headers = {'Content-type':'application/json'}
request = urllib2.Request(self._address, data=body, headers=headers)
# make request to server
try:
response = urllib2.urlopen(request, timeout=self._timeout)
except Exception:
logger.warn('error making jsonrpc request', exc_info=True)
raise
# translate response from json
try:
cb_data = response.read()
data = json_decode(cb_data)
except Exception:
try:
logger.info('json data: %r', cb_data)
except Exception:
pass
raise
if 'error' in data:
try:
raise Exception('Error %r: %r %r'%data['error'])
except Exception:
raise Exception('Error %r'%data['error'])
if 'result' in data:
return data['result']
else:
return None
async def post(self):
try:
""" Send notifications """
if not self.can("send_notification"):
self.send_response(
FORBIDDEN,
dict(error="No permission to send notification"))
return
# if request body is json entity
request_dict = json_decode(self.request.body)
# application specific request_dict
extra = request_dict.get("extra", {})
# Hook
if "processor" in extra:
try:
proc = import_module("hooks." + extra["processor"])
request_dict = proc.process_pushnotification_payload(
request_dict)
except Exception as ex:
logging.error(str(ex))
self.send_response(BAD_REQUEST, dict(error=str(ex)))
return
if not self.token:
self.token = request_dict.get("token", None)
device = request_dict.get("device", DEVICE_TYPE_FCM).lower()
channel = request_dict.get("channel", "default")
alert = request_dict.get("alert", "")
token = self.dao.find_token(self.token)
if not token:
token = EntityBuilder.build_token(self.token, device,
self.appname, channel)
if not self.can("create_token"):
self.send_response(
BAD_REQUEST,
dict(
error=
"Unknow token and you have no permission to create"
),
)
return
try:
# TODO check permission to insert
self.dao.add_token(token)
except Exception as ex:
logging.error(str(ex))
self.send_response(INTERNAL_SERVER_ERROR,
dict(error=str(ex)))
return
logging.info("sending notification to %s: %s" %
(device, self.token))
# if device in [DEVICE_TYPE_FCM, DEVICE_TYPE_ANDROID]:
if device.endswith(DEVICE_TYPE_FCM):
fcm = request_dict.get("fcm", {})
try:
fcmconn = self.fcmconnections[self.app["shortname"]][0]
await fcmconn.process(token=self.token,
alert=alert,
fcm=fcm)
except Exception as ex:
statuscode = ex.code
# reference:
# https://firebase.google.com/docs/reference/fcm/rest/v1/ErrorCode
response_json = json_decode(ex.response.body)
logging.error(response_json)
self.add_to_log(
"error", tornado.escape.to_unicode(ex.response.body))
self.send_response(statuscode,
dict(error="error response from fcm"))
return
elif device == DEVICE_TYPE_IOS:
# Use splitlines trick to remove line ending (only for iOS).
if type(alert) is not dict:
alert = "".join(alert.splitlines())
# https://developer.apple.com/library/archive/documentation/NetworkingInternet/Conceptual/RemoteNotificationsPG/PayloadKeyReference.html#//apple_ref/doc/uid/TP40008194-CH17-SW1
apns_default = {
"badge": None,
"sound": "default",
"push_type": "alert"
}
apnspayload = request_dict.get("apns",
request_dict.get("fcm", {}))
conn = self.get_apns_conn()
if conn:
try:
conn.process(
token=self.token,
alert=alert,
apns={
**apns_default,
**apnspayload
},
)
except Exception as ex:
logging.error(ex)
self.reattempt_push_apns(alert, apnspayload,
self.token)
return
else:
logging.error("no active apns connection")
elif device == DEVICE_TYPE_WNS:
request_dict.setdefault("wns", {})
wns = self.wnsconnections[self.app["shortname"]][0]
wns.process(
token=request_dict["token"],
alert=alert,
extra=extra,
wns=request_dict["wns"],
)
else:
logging.error("invalid device type %s" % device)
self.send_response(BAD_REQUEST,
dict(error="Invalid device type"))
return
logmessage = "payload: %s, access key: %s" % (
tornado.escape.to_unicode(self.request.body),
self.appkey,
)
self.add_to_log("notification", logmessage)
self.send_response(ACCEPTED)
except Exception as ex:
traceback_ex = traceback.format_exc()
logging.error("%s %s" % (traceback_ex, str(ex)))
self.send_response(INTERNAL_SERVER_ERROR, dict(error=str(ex)))
def setconfig(self, key, value):
"""Set a configuration variable. 'value' may be a string or a Python expression."""
if key not in ('rpcuser', 'rpcpassword'):
value = json_decode(value)
self.config.set_key(key, value)
return True
def from_json(cls, jsonstr):
return cls.from_dict(json_decode(jsonstr))
def handle_read(self):
raw = self.recv(8192)
if raw:
msg = util.json_decode(json.loads(raw))
self.dest.queue(msg)
self.buffer = {'result': 'done'}