def test_save(self):
config = Configuration(self.config_path + ".yaml")
config.save(self.config_path + ".bak")
if os.path.isfile(self.config_path + ".bak"):
os.remove(self.config_path + ".bak")
self.assertTrue(True)
else:
self.assertFalse(True)
def test_load_config(self):
config = Configuration()
with self.assertRaises(Exception) as context:
config.load_config(self.config_path + ".ini")
self.assertTrue("Configuration file type unknown or not supported: "
"%s%stestfile_config.ini" % \
(os.path.dirname(os.path.abspath(__file__)), os.sep)
in str(context.exception))
def lambda_handler(event, context):
# Get the object from the event and show its content type
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'])
try:
print("File to process %s" % key)
response = s3.get_object(Bucket=bucket, Key=key)
body = response['Body']
data = body.read()
# If the name has a .gz extension, then decompress the data
if key[-3:] == '.gz':
data = zlib.decompress(data, 16 + zlib.MAX_WBITS)
config = Configuration("config.json")
con = Sender(config=config.get("sender"))
# Send json events to Devo
print("Starting to send lines to Devo")
counter = 0
for line in data.splitlines():
events_json = json.loads(line)
for single_event in events_json["Records"]:
dumped_event = json.dumps(single_event)
aws_account_id = "a"
if single_event.get("getuserIdentity", None) is not None:
if single_event.get("getuserIdentity")\
.get("accountId", None) is not None:
aws_account_id = single_event["getuserIdentity"][
"accountId"]
elif single_event.get("account", None) is not None:
aws_account_id = single_event["account"]
elif single_event.get("recipientAccountId", None) is not None:
aws_account_id = single_event["recipientAccountId"]
aws_region = "b"
if single_event.get("awsRegion", None) is not None:
aws_region = single_event["awsRegion"]
elif single_event.get("region", None) is not None:
aws_region = single_event["region"]
tag = "{!s}.{!s}.{!s}".format(config.get("tag"),
aws_account_id, aws_region)
counter += con.send(tag=encode(tag),
msg=encode(dumped_event),
zip=False)
con.close()
print("Finished sending lines to Devo (%d)" % counter)
except Exception as e:
print(e)
print("Error getting file '%s' from bucket '%s'. Make sure they \
exist and your bucket is in the same region as this function." %
(key, bucket))
def test_load_yaml(self):
config = Configuration()
config.load_yaml(self.config_path + ".yaml")
self.assertDictEqual(config.cfg, {
"devo": {
"die": "hard"
},
"api": {
"velazquez": "Then I am beautiful?"
}
})
def test_mix_json(self):
config = Configuration(self.config_path + ".json")
config.mix({"test": "ok"})
self.assertDictEqual(
config.cfg, {
"devo": {
"die": "hard"
},
"api": {
"velazquez": "Then I am beautiful?"
},
"test": "ok"
})
def setUp(self):
self.query = 'from demo.ecommerce.data select * limit 1'
self.app_name = "testing-app_name"
self.uri = os.getenv('DEVO_API_ADDRESS',
'https://apiv2-us.devo.com/search/query')
self.key = os.getenv('DEVO_API_KEY', None)
self.secret = os.getenv('DEVO_API_SECRET', None)
self.token = os.getenv('DEVO_AUTH_TOKEN', None)
self.query_id = os.getenv('DEVO_API_QUERYID', None)
self.user = os.getenv('DEVO_API_USER', "python-sdk-user")
self.comment = os.getenv('DEVO_API_COMMENT', None)
configuration = Configuration()
configuration.set(
"api", {
"query": self.query,
"address": self.uri,
"key": self.key,
"secret": self.secret,
"token": self.token,
"query_id": self.query_id,
"user": self.user,
"comment": self.comment,
"app_name": self.app_name
})
self.config_path = "/tmp/devo_api_tests_config.json"
configuration.save(path=self.config_path)
def setUp(self):
self.address = os.getenv('DEVO_SENDER_SERVER', "0.0.0.0")
self.port = int(os.getenv('DEVO_SENDER_PORT', 4488))
self.tcp_address = os.getenv('DEVO_SENDER_TCP_SERVER', "0.0.0.0")
self.tcp_port = int(os.getenv('DEVO_SENDER_TCP_PORT', 4489))
self.key = os.getenv('DEVO_SENDER_KEY', CLIENT_KEY)
self.cert = os.getenv('DEVO_SENDER_CERT', CLIENT_CERT)
self.chain = os.getenv('DEVO_SENDER_CHAIN', CLIENT_CHAIN)
self.local_key = os.getenv(CLIENT_KEY)
self.test_tcp = os.getenv('DEVO_TEST_TCP', "True")
self.my_app = 'test.drop.free'
self.my_bapp = b'test.drop.free'
self.my_date = 'my.date.test.sender'
self.test_file = "".join((os.path.dirname(os.path.abspath(__file__)),
os.sep, "testfile_multiline.txt"))
self.test_msg = 'Test send msg\n'
self.localhost = socket.gethostname()
# change this value if you want to send another number of test string
self.default_numbers_sendings = 10
configuration = Configuration()
configuration.set(
"sender", {
"key": self.key,
"cert": self.cert,
"chain": self.chain,
"address": self.address,
"port": self.port,
})
self.config_path = "/tmp/devo_sender_tests_config.json"
configuration.save(path=self.config_path)
def lambda_handler(event, context):
# Get the object from the event and show its content type
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'])
try:
print("File to process %s" % key)
response = s3.get_object(Bucket=bucket, Key=key)
body = response['Body']
data = body.read()
###### START: From this point until END, you need to
###### carefully review the code to make sure all
###### variables match your environment.
# If the name has a .gz extension, then decompress the data
if key[-3:] == '.gz':
data = zlib.decompress(data, 16 + zlib.MAX_WBITS)
config = Configuration("config.json")
con = Sender(config=config.get("sender"))
# Send json events to Devo
print("Starting to send lines to Devo")
counter = 0
for line in data.splitlines():
events_json = json.loads(line)
for single_event in events_json["Records"]:
counter += con.send(tag=encode(config.get("tag")),
msg=encode(json.dumps(single_event)),
zip=False)
con.close()
print("Finished sending lines to Devo (%d)" % counter)
###### END of code containing key variables.
except Exception as e:
print(e)
print("Error getting file '%s' from bucket '%s'. Make sure they \
exist and your bucket is in the same region as this function." %
(key, bucket))
def init_conf(args):
""" Generic configuration of CLI, from config file and cli arguments """
config = Configuration()
try:
if args.get('config'):
config.load_config(args.get('config'), 'sender')
if args.get('env'):
config.set("address",
os.environ.get('DEVO_SENDER_ADDRESS',
os.environ.get('DEVO_SENDER_URL', None)))
config.set("port", os.environ.get('DEVO_SENDER_PORT', None))
config.set("key", os.environ.get('DEVO_SENDER_KEY', None))
config.set("cert", os.environ.get('DEVO_SENDER_CERT', None))
config.set("chain", os.environ.get('DEVO_SENDER_CHAIN', None))
if args.get('default'):
config.load_default_config(section="sender")
finally:
config.mix(dict(args))
return config
def configure(args):
"""For load configuration file/object"""
if args.get('config'):
config = Configuration(args.get('config'))
config.mix(dict(args))
else:
config = dict(args)
if 'freq' in config.keys():
parts = config['freq'].split('-')
config['freq'] = (float(parts[0]), float(parts[1]))
config['template'] = config['template'].read()
# Initialize LtSender with the config credentials but only
# if we aren't in batch mode or simulation mode
engine = None
if not (config['batch_mode'] or config['simulation']):
try:
if "sender" not in config.keys():
config['sender'] = {
'key': config.get('key', None),
'chain': config.get('chain', None),
'cert': config.get('cert', None),
'address': config.get('address', None),
'port': config.get('port', 443)
}
engine = Sender(config=config.get('sender'))
except Exception as error:
print_error(error, show_help=False)
print_error("Error when loading devo sender configuration",
show_help=True)
return engine, config
def init_conf(args):
""" Generic configuration of CLI, from config file and cli arguments """
config = Configuration()
if args.get('config'):
config.load_json(args.get('config'), 'sender')
config.mix(dict(args))
if "address" not in args.keys() and "url" in args.keys():
config.set('address', args['url'])
if "address" not in args.keys():
address = os.environ.get('DEVO_SENDER_ADDRESS', None)
if not address:
address = os.environ.get('DEVO_SENDER_URL', None)
config.set("address", address)
config.set("port", os.environ.get('DEVO_SENDER_PORT', None))
if config.get()['cert_reqs']:
if "config" not in args.keys() and "key" not in args.keys() \
and "cert" not in args.keys() and "chain" not in args.keys():
config.set("key", os.environ.get('DEVO_SENDER_KEY', None))
config.set("cert", os.environ.get('DEVO_SENDER_CERT', None))
config.set("chain", os.environ.get('DEVO_SENDER_CHAIN', None))
if not config.keys("key") and not config.keys("cert") and not \
config.keys("chain") and os.path.exists("~/.devo.json"):
config.load_default_json('api')
config.keys('from')
config.keys('to')
return config
def configure(args):
"""For load configuration file/object"""
if args.get('config'):
config = Configuration(path=args.get('config'), section="faker")
config.mix(dict(args))
else:
config = dict(args)
if 'frequency' in config.keys() and isinstance(config.get("frequency"),
(str, bytes)):
config['frequency'] = tuple(
[float(x) for x in config.get("frequency").split("-")])
config['template'] = config['template'].read()
# Initialize devo.sender with the config credentials but only
# if we aren't in batch mode or simulation mode
engine = None
if not (config['batch_mode'] or config['simulation']
or config.get('file_name', None)):
try:
if "sender" not in config.keys():
config['sender'] = {
'key': config.get('key', None),
'chain': config.get('chain', None),
'cert': config.get('cert', None),
'address': config.get('address', None),
'port': config.get('port', 443)
}
engine = Sender(config=config.get('sender'))
except Exception as error:
print_error(error, show_help=False)
print_error("Error when loading devo sender configuration",
show_help=True)
return engine, config
def configure(args):
"""
Load CLI configuration
:param args: args from files, launch vars, etc
:return: Client API Object and Config values in array
"""
config = Configuration()
try:
if args.get('config'):
config.load_config(args.get('config'), 'api')
if args.get('env'):
config.set("key", os.environ.get('DEVO_API_KEY', None))
config.set("secret", os.environ.get('DEVO_API_SECRET', None))
config.set("url", os.environ.get('DEVO_API_URL', None))
config.set("user", os.environ.get('DEVO_API_USER', None))
config.set("comment", os.environ.get('DEVO_API_COMMENT', None))
if args.get('default'):
config.load_default_config(section="api")
finally:
config.mix(dict(args))
conf = config.get()
# Try to compose the api
api = None
try:
api = Client.from_config(conf)
except DevoClientException as error:
print_error(str(error), show_help=True)
return api, conf
def test_get_keys_chain_in_array(self):
config = Configuration(self.config_path + ".yaml")
self.assertEqual(config.get(["devo", "die"]), "hard")
def test_key_exist(self):
config = Configuration(self.config_path + ".json")
self.assertFalse(config.key_exist("noexiste"))
self.assertTrue(config.key_exist("api"))
def test_load_section_yaml(self):
config = Configuration(self.config_path + ".yaml", "devo")
self.assertDictEqual(config.cfg, {"die": "hard"})
def test_add_key(self):
config = Configuration(self.config_path + ".yaml")
config.set("logtrust", "old")
self.assertEqual(config["logtrust"], "old")
def test_load_section_json(self):
config = Configuration(self.config_path + ".json", "api")
self.assertDictEqual(config.cfg, {"velazquez": "Then I am beautiful?"})
def test_add_key_chain(self):
config = Configuration(self.config_path + ".yaml")
config.set(["devo", "old", "name"], "logtrust")
self.assertEqual(config["devo"]['old']['name'], "logtrust")
self.assertEqual(config.get("devo", 'old', 'name'), "logtrust")
def configure(args):
"""
Load CLI configuration
:param args: args from files, launch vars, etc
:return: Clien t API Object and Config values in array
"""
config = Configuration()
try:
if args.get('config'):
config.load_config(args.get('config'), 'api')
if args.get('env'):
config.set("key", os.environ.get('DEVO_API_KEY', None))
config.set("secret", os.environ.get('DEVO_API_SECRET', None))
config.set("token", os.environ.get('DEVO_API_TOKEN', None))
config.set("jwt", os.environ.get('DEVO_API_JWT', None))
config.set("address", os.environ.get('DEVO_API_ADDRESS', None))
config.set("user", os.environ.get('DEVO_API_USER', None))
config.set("comment", os.environ.get('DEVO_API_COMMENT', None))
config.set("retries", os.environ.get('DEVO_API_RETRIES', None))
config.set("timeout", os.environ.get('DEVO_API_TIMEOUT', None))
if args.get('default'):
config.load_default_config(section="api")
except Exception as error:
print_error(str(error), show_help=True)
finally:
config.mix(dict(args))
# Try to compose the api
try:
api = Client(config=config)
except DevoClientException as error:
print_error(str(error), show_help=True)
if isinstance(error, DevoClientException):
raise DevoClientException(error.args[0])
else:
raise DevoClientException(str(error.args[0]))
return api, config
def configure(args):
"""
Load CLI configuration
:param args: args from files, launch vars, etc
:return: Client API Object and Config values in array
"""
config = Configuration()
if args.get('config') != "~/.devo.json":
config.load_json(args.get('config'), 'api')
config.mix(dict(args))
if "key" not in args.keys() and "api" not in args.keys() \
and "token" not in args.keys():
config.set("key", os.environ.get('DEVO_API_KEY', None))
config.set("secret", os.environ.get('DEVO_API_SECRET', None))
if "url" not in args.keys():
config.set("url", os.environ.get('DEVO_API_URL', None))
if not config.keys("key") and not config.keys("api") \
and not config.keys("token") \
and os.path.exists("~/.devo.json"):
config.load_default_json('api')
config.keys('from')
config.keys('to')
# Try to compose the api
api = None
try:
api = Client.from_config(config.get())
except DevoClientException as error:
print_error(str(error), show_help=True)
return api, config.get()