def main():
account_id = '*****@*****.**'
password = b'asdfasdf'
temp_config_path = os.path.join(os.getcwd(), 'test.data', 'secrets')
test_vault_path = os.path.join(os.getcwd(), 'test.data', 'vaults')
name = 'test-vault'
cab = Cabinet(account_id, password, temp_config_path)
cab.open(name, test_vault_path)
item = get_item()
cab.add(item)
item = get_item(42)
cab.add(item)
print('-' * 10)
print("Result for cabinet.get_all():")
import pprint
pprint.pprint(cab.get_all())
print('-' * 10)
name = 'test-item #42'
print("Result for cabinet.get(name):")
print(name)
pprint.pprint(cab.get(name))
def main():
account_id = '*****@*****.**'
password = b'asdfasdf'
temp_config_path = os.path.join(os.getcwd(), 'test.data', 'secrets')
test_vault_path = os.path.join(os.getcwd(), 'test.data', 'vaults')
name = 'test-vault'
cab = Cabinet(account_id, password, temp_config_path)
cab.open(name, test_vault_path)
item = get_item()
cab.add(item)
item = get_item(42)
cab.add(item)
print('-'*10)
print("Result for cabinet.get_all():")
import pprint
pprint.pprint(cab.get_all())
print('-'*10)
name = 'test-item #42'
print("Result for cabinet.get(name):")
print(name)
pprint.pprint(cab.get(name))
def setUp(self):
account_id = '*****@*****.**'
password = b'asdfasdf'
self._base_dir = base_dir = os.path.join(os.getcwd(), 'tmp.tests')
temp_config_path = os.path.join(base_dir, 'secrets')
test_vault_path = os.path.join(base_dir, 'vaults')
name = 'test-vault'
cab = Cabinet(account_id, password, temp_config_path)
cab.open(name, test_vault_path)
self.cab = cab
def setUp(self):
account_id = '*****@*****.**'
password = b'asdfasdf'
self._base_dir = base_dir = os.path.join(os.getcwd(), 'tmp.tests')
temp_config_path = os.path.join(base_dir, 'secrets')
test_vault_path = os.path.join(base_dir, 'vaults')
name = 'test-vault'
cab = Cabinet(account_id, password, temp_config_path)
cab.open(name, test_vault_path)
self.cab = cab
def open_vault(username, password, vault_name, vault_path=None):
# TODO: Implement username/password/vault validation and opening
global vault_config
config_path = vault_config.get('config_path')
if vault_path is None:
vault_path = vault_config.get('vault_path')
cab = Cabinet(username, password, config_path)
cab.open(vault_name, vault_path)
vault_config['cabinet'] = cab
session['username'] = username
session['password'] = password
session['vault_path'] = vault_path
return {'success': True} # TODO: don't hardcode this
def on_receive(self, payload):
try:
# logger.debug(
# "SubscriberProcessorService.process_subscriber: "
# f"processing subscriber: {payload['subscriber']}"
# )
# start_time = time.time()
redis_client = redis.Redis(connection_pool=REDIS_POOL,
socket_timeout=1)
cab = Cabinet(settings.CABINET_URL)
cached_cabinet = CachedCabinet(
cab, RedisEngine(redis_client, prefix="CABINET_CACHE", ttl=30))
general_settings = cached_cabinet.general()
limit = general_settings["push_limit_per_token"]
bid_interval = general_settings["token_bid_interval"]
token = payload["subscriber"]["_id"]
subscriber_pushes = redis_client.get(
f"subscriber.pushes.count:{token}")
try:
subscriber_pushes = int(subscriber_pushes)
except TypeError:
subscriber_pushes = 0
# subscriber_pushes = (self.counter_service
# .get_pushes_count(token))
has_quota = subscriber_pushes < limit
last_bid_key = f"subscriber:{token}:last-bid-at"
try:
last_bid_time = int(redis_client.get(last_bid_key))
except TypeError:
last_bid_time = None
if last_bid_time:
time_passed = time.time() - last_bid_time
time_passed_enough = time_passed > (bid_interval * 60)
logger.debug("SubscriberProcessorService.process_subscriber: "
f"passed time since last bid {time_passed}")
else:
time_passed_enough = True
if has_quota and time_passed_enough:
send_to_ssp(payload)
# self.queue.publish.call_async(payload)
redis_client.set(last_bid_key,
int(time.time()),
ex=DAY_SECONDS)
else:
logger.debug("SubscriberProcessorService.process_subscriber: "
f"for subscriber: {payload['subscriber']['_id']} "
f"has_quota={has_quota} "
f"time_passed_enough={time_passed_enough}")
# finish_time = time.time()
# logger.debug(
# "SubscriberProcessorService.process_subscriber: "
# "total execution time "
# f"{(finish_time - start_time) * 1000}ms"
# )
except Exception as e:
logger.error(f"SubscriberProcessor: exception {e}")
def open_vault(self, account_id, password, vault_name):
"""
Open the vault identified by the vault_name.
Params:
:param account_id:
:type: String
:param password:
:type: String
:param vault_name: The name of the vault
:type: String
TODO: Implement account_id/password/vault validation and opening
"""
self.cab = Cabinet(account_id, password, self.secrets_path)
self.cab.open(vault_name, self.vault_path)
# TODO: Add open check. For this, the vault should verify keys
return True
def run(self):
logger.debug('campaign_processor_service.run')
redis_client = redis.Redis(connection_pool=REDIS_POOL,
socket_timeout=1)
cab = Cabinet(settings.CABINET_URL)
cached_cabinet = CachedCabinet(
cab, RedisEngine(redis_client, prefix="CABINET_CACHE", ttl=5))
campaigns = cached_cabinet.campaigns()
logger.debug("CampaignsRunnerService.run: "
f"received active campaigns {campaigns}")
for campaign in campaigns:
(self.campaign_processor_service.process_campaign.call_async(
campaign))
def process_campaign(self, payload):
logger.info("CampaignProcessorService.process_campaign: "
f"processing campaign - {payload}")
start_time = time.time()
# @todo #1:15min daily count check
# total_limit = payload['total_limit']
# total_count = (self.stats_service
# .get_pushes_total_count(payload["id"]))
# daily_count = (self.stats_service
# .get_pushes_daily_count(payload["id"]))
# if total_count >= total_limit or daily_count >= total_limit:
# print("CampaignProcessorService.process_campaign: "
# f"campaign limit exceeded: {payload}")
# return None
targetings = payload["targetings"]
redis_client = redis.Redis(connection_pool=REDIS_POOL,
socket_timeout=1)
cab = Cabinet(settings.CABINET_URL)
cached_cabinet = CachedCabinet(
cab, RedisEngine(redis_client, prefix="CABINET_CACHE", ttl=30))
cabinet_settings = cached_cabinet.general()
start_hour = cabinet_settings["start_hour"]
end_hour = cabinet_settings["end_hour"]
hours_whitelist = list(range(start_hour, end_hour + 1))
volume = payload["subscriber_selection_size"]
with Timer() as t:
subscribers = (get_subscribers(targetings, hours_whitelist,
volume))
logger.debug("CampaignProcessorService.process_campaign: "
"Request to get subsctiber took %.03f sec." % t.interval)
if not subscribers:
logger.info("CampaignProcessorService.process_campaign: "
f"no subscribers found for campaign: #{payload['id']}")
return
for subscriber in subscribers:
# time1 = time.time()
message = dict(campaign=payload, subscriber=subscriber)
subscriber_processor_ref.tell(message)
# (self.subscriber_processor_service.process_subscriber
# .call_async(dict(campaign=payload, subscriber=subscriber)))
# time2 = time.time()
# logger.debug("CampaignProcessorService.process_campaign: "
# "called process_subscriber in "
# f"{int((time2 - time1) * 1000)}ms")
end_time = time.time()
logger.info("CampaignProcessorService.process_campaign: "
f"for campaign #{payload['id']} "
f"processed {len(subscribers)} subscribers "
f"in {(end_time - start_time) * 1000}ms")
class CabinetWrapper:
"""
A wrapper to easely use the cabinet library.
TODO: In the future we should try moving this all to the library.
"""
def __init__(self, vault_name=None, account_id=None):
"""
Initialize the cabinet instance using the given vault/account or from
the configuration file if you don't specify them.
This will prompt for the account password to open the vault.
:param vault_name: The name of the vault
:type: str
:param account_id:
:type: str
"""
join = os.path.join
self.base_path = join(os.path.expanduser('~'), '.config', 'cabinet')
self.config_file = join(self.base_path, 'cli.ini')
self.secrets_path = join(self.base_path, 'secrets')
self.vault_path = join(self.base_path, 'vaults')
self._ready = self._load_credentials(vault_name, account_id)
def _load_credentials(self, vault_name=None, account_id=None):
"""
It loads the vault name and account id from the configuration, then
it overrides the configuration with the cli options (if entered).
Finally, it prompts for the account password and opens the vault.
Params:
:param vault_name: The name of the vault
:type: String
:param account_id:
:type: String
TODO: Move the print and die to utils
"""
config = get_configs(self.config_file)
if vault_name:
self.vault_name = vault_name
elif config.get('vault_name'):
self.vault_name = config.get('vault_name')
else:
print('Vault not specified')
exit()
if account_id:
self.account_id = account_id
elif config.get('account_id'):
self.account_id = config.get('account_id')
else:
print('Account not specified')
exit()
self.password = getpass()
return self.open_vault(self.account_id, self.password, self.vault_name)
def open_vault(self, account_id, password, vault_name):
"""
Open the vault identified by the vault_name.
Params:
:param account_id:
:type: String
:param password:
:type: String
:param vault_name: The name of the vault
:type: String
TODO: Implement account_id/password/vault validation and opening
"""
self.cab = Cabinet(account_id, password, self.secrets_path)
self.cab.open(vault_name, self.vault_path)
# TODO: Add open check. For this, the vault should verify keys
return True
def get_by_tags(self, tags):
"""
Get all the items from the vault without the 'content' (i.e: only name
and tags) that matches all the given tags.
:returns: The list of items
:type: List of Dictionaries.
"""
return self.cab.get_by_tags(tags)
def get_tags(self):
"""
Get all the tags from the vault.
"""
tags = self.cab.get_tags()
print("Tags:", ','.join(tags))
def get_all(self):
"""
Get all the items from the vault without the 'content' (i.e: only name
and tags).
:returns: The list of items
:type: List of Dictionaries.
"""
return self.cab.get_all()
def get_item(self, name, print_all):
"""
Get an item from the vault.
:param name: The name of the item to recover.
:type: String
:param print_all: Print all the information related to the item.
:type: Boolean
:returns: The item with the specified name.
:type: Dictionary
"""
if self._ready:
item = self.cab.get(name)
if not item:
print('Item with name "{0}" not found!'.format(name))
return
if print_all:
print("Name: {0}\nTags: {1}\nContent: {2}".format(
name, item['tags'], item['content']))
else:
print(item['content'])
def add_item(self, name, tags, content):
"""
Add an item to the vault.
:param item: The item's name
:type: str
:param tags: The item's tags
:type: list
:param content: The item's contents
:type: any
"""
if self._ready:
item = {'name': name, 'tags': tags, 'content': content}
self.cab.add(item)
def update(self, name, tags, content):
if self._ready:
self.cab.update(name, content, tags)
def rename(self, name, new_name):
if self._ready:
self.cab.rename(name, new_name)
def search(self, tags, show_tags):
"""
Search for items with the given tags.
:param tags: the tags to search for
:type tags: list
:param show_tags: whether we should show the tags or not
:type show_tags: bool
"""
if self._ready:
item_list = self.get_by_tags(tags)
print("The following items were found:")
tag_tpl = " tagged with {1}" if show_tags else ''
for item in item_list:
print(('\t-"{0}"' + tag_tpl).format(item['name'],
item['tags']))