class PurgeModule(MigrationModule):
CONF_PATH = 'etc/migration/purge.conf'
CATEGORY = 'PURGE'
def __init__(self, collections=None, *args, **kwargs):
super(PurgeModule, self).__init__(*args, **kwargs)
self.logger = Logger.get('migrationmodule', MigrationModule.LOG_PATH)
self.config = Configuration.load(PurgeModule.CONF_PATH, Json)
conf = self.config.get(self.CATEGORY, {})
self.storage = Storage(account=Account(user='******', group='root'))
if collections is not None:
self.collections = collections
else:
self.collections = conf.get('collections', DEFAULT_COLLECTIONS)
def init(self, yes=False):
for collection in self.collections:
self.logger.info(u'Drop collection: {0}'.format(collection))
self.storage.drop_namespace(collection)
def update(self, yes=False):
pass
class PurgeModule(MigrationModule):
@property
def collections(self):
if not hasattr(self, '_collections'):
self.collections = None
return self._collections
@collections.setter
def collections(self, value):
if value is None:
value = []
self._collections = value
def __init__(self, collections=None, *args, **kwargs):
super(PurgeModule, self).__init__(*args, **kwargs)
self.storage = Storage(account=Account(user='******', group='root'))
if collections is not None:
self.collections = collections
def init(self):
for collection in self.collections:
self.logger.info('Drop collection: {0}'.format(collection))
self.storage.drop_namespace(collection)
def update(self):
pass
class PurgeModule(MigrationModule):
CONF_PATH = 'etc/migration/purge.conf'
CATEGORY = 'PURGE'
def __init__(self, collections=None, *args, **kwargs):
super(PurgeModule, self).__init__(*args, **kwargs)
self.logger = Logger.get('migrationmodule', MigrationModule.LOG_PATH)
self.config = Configuration.load(PurgeModule.CONF_PATH, Json)
conf = self.config.get(self.CATEGORY, {})
self.storage = Storage(account=Account(user='******', group='root'))
if collections is not None:
self.collections = collections
else:
self.collections = conf.get('collections', DEFAULT_COLLECTIONS)
def init(self, yes=False):
for collection in self.collections:
self.logger.info(u'Drop collection: {0}'.format(collection))
self.storage.drop_namespace(collection)
def update(self, yes=False):
pass
def __init__(self, prompt, account, namespace='object', crecord_type=None):
super(Browser, self).__init__(prompt)
self.account = account
self.namespace = namespace
self.crecord_type = crecord_type
self.storage = Storage(account,
namespace=namespace,
logging_level=INFO)
def test_01_Init(self):
global STORAGE
STORAGE = Storage(self.user_account,
namespace='unittest',
logging_level=DEBUG)
records = STORAGE.find(account=self.root_account)
STORAGE.remove(records, account=self.root_account)
def build(self):
consos = self.data
storage = Storage(self.root_account, namespace=self.COLLECTION, logging_level=DEBUG)
for conso in consos:
serie = self.factory(conso)
# Store the consolidation into the database
current_record = Record(serie, storage=storage)
storage.put(current_record)
def __init__(self, collections=None, *args, **kwargs):
super(PurgeModule, self).__init__(*args, **kwargs)
self.logger = Logger.get('migrationmodule', MigrationModule.LOG_PATH)
self.config = Configuration.load(PurgeModule.CONF_PATH, Json)
conf = self.config.get(self.CATEGORY, {})
self.storage = Storage(account=Account(user='******', group='root'))
if collections is not None:
self.collections = collections
else:
self.collections = conf.get('collections', DEFAULT_COLLECTIONS)
def __init__(self, collections=None, *args, **kwargs):
super(PurgeModule, self).__init__(*args, **kwargs)
self.storage = Storage(account=Account(user='******', group='root'))
if collections is not None:
self.collections = collections
def __init__(self, prompt, account, namespace='object', crecord_type=None):
super(Browser, self).__init__(prompt)
self.account = account
self.namespace = namespace
self.crecord_type = crecord_type
self.storage = Storage(
account, namespace=namespace, logging_level=INFO)
def __init__(self, collections=None, *args, **kwargs):
super(PurgeModule, self).__init__(*args, **kwargs)
self.logger = Logger.get('migrationmodule', MigrationModule.LOG_PATH)
self.config = Configuration.load(PurgeModule.CONF_PATH, Json)
conf = self.config.get(self.CATEGORY, {})
self.storage = Storage(account=Account(user='******', group='root'))
if collections is not None:
self.collections = collections
else:
self.collections = conf.get('collections', DEFAULT_COLLECTIONS)
class Browser(Cmd):
def __init__(self, prompt, account, namespace='object', crecord_type=None):
super(Browser, self).__init__(prompt)
self.account = account
self.namespace = namespace
self.crecord_type = crecord_type
self.storage = Storage(account,
namespace=namespace,
logging_level=INFO)
def do_ls(self, crecord_type=None):
if self.crecord_type:
records = self.storage.find({'crecord_type': self.crecord_type})
elif crecord_type:
records = self.storage.find({'crecord_type': crecord_type})
else:
records = self.storage.find()
self.print_records(records)
def do_cat(self, _id):
try:
if _id != '*':
record = self.storage.get(_id)
record.cat()
except Exception as err:
print("Impossible to cat {0}: {1}".format(_id, err))
def do_dump(self, _id):
try:
if _id != '*':
record = self.storage.get(_id)
record.cat(dump=True)
except Exception as err:
print("Impossible to dump {0}: {1}".format(_id, err))
def do_rm(self, _id):
try:
self.storage.remove(_id)
except Exception as err:
print("Impossible to remove {0}: {1}".format(_id, err))
def do_cd(self, path):
if path == "..":
return True
def print_records(self, records):
print("Total: {0}".format(len(records)))
lines = []
for record in records:
line = []
line.append(record.owner)
line.append(record.group)
line.append(str(sys.getsizeof(record)))
date = datetime.fromtimestamp(record.write_time)
line.append(str(date))
line.append(record.type)
line.append(str(record._id))
line.append(str(record.name))
# self.columnize(line, displaywidth=200)
lines.append(line)
# Quick and dirty ...
max_ln = {}
for line in lines:
i = 0
for word in line:
try:
if len(word) > max_ln[i]:
max_ln[i] = len(word)
except Exception:
max_ln[i] = len(word)
i += 1
# new_lines = []
for line in lines:
i = 0
new_line = ""
for word in line:
empty = ""
nb = max_ln[i] - len(word)
for s in range(nb + 2):
empty += " "
new_line += word + empty
i += 1
print(new_line)
def test_01_Init(self):
global STORAGE
STORAGE = Storage(self.user_account, namespace='unittest', logging_level=DEBUG)
records = STORAGE.find(account=self.root_account)
STORAGE.remove(records, account=self.root_account)
raise Exception('Invalid cryptedKey ... (%s)' % authkey)
def test_04_authkey(self):
ACCOUNT = Account(user="******", group="capensis")
authkey = ACCOUNT.get_authkey()
if not authkey:
raise Exception('Invalid authkey ... (%s)' % authkey)
def test_05_Store(self):
ACCOUNT = Account(user="******", group="capensis")
STORAGE.put(ACCOUNT)
def test_09_Remove(self):
# Anonymous cant remove account
self.assertRaises(ValueError, STORAGE.remove, ACCOUNT, Account())
# But root can ;)
STORAGE.remove(ACCOUNT)
def test_99_DropNamespace(self):
STORAGE.drop_namespace('unittest')
if __name__ == "__main__":
STORAGE = Storage(Account(user="******", group="root"), namespace='unittest')
output = root_path + "/tmp/tests_report"
unittest.main(testRunner=xmlrunner.XMLTestRunner(output=output),
verbosity=3)
class Browser(Cmd):
def __init__(self, prompt, account, namespace='object', crecord_type=None):
super(Browser, self).__init__(prompt)
self.account = account
self.namespace = namespace
self.crecord_type = crecord_type
self.storage = Storage(
account, namespace=namespace, logging_level=INFO)
def do_ls(self, crecord_type=None):
if self.crecord_type:
records = self.storage.find({'crecord_type': self.crecord_type})
elif crecord_type:
records = self.storage.find({'crecord_type': crecord_type})
else:
records = self.storage.find()
self.print_records(records)
def do_cat(self, _id):
try:
if _id != '*':
record = self.storage.get(_id)
record.cat()
except Exception as err:
print("Impossible to cat {0}: {1}".format(_id, err))
def do_dump(self, _id):
try:
if _id != '*':
record = self.storage.get(_id)
record.cat(dump=True)
except Exception as err:
print("Impossible to dump {0}: {1}".format(_id, err))
def do_rm(self, _id):
try:
self.storage.remove(_id)
except Exception as err:
print("Impossible to remove {0}: {1}".format(_id, err))
def do_cd(self, path):
if path == "..":
return True
def print_records(self, records):
print("Total: {0}".format(len(records)))
lines = []
for record in records:
line = []
line.append(record.owner)
line.append(record.group)
line.append(str(sys.getsizeof(record)))
date = datetime.fromtimestamp(record.write_time)
line.append(str(date))
line.append(record.type)
line.append(str(record._id))
line.append(str(record.name))
# self.columnize(line, displaywidth=200)
lines.append(line)
# Quick and dirty ...
max_ln = {}
for line in lines:
i = 0
for word in line:
try:
if len(word) > max_ln[i]:
max_ln[i] = len(word)
except:
max_ln[i] = len(word)
i += 1
# new_lines = []
for line in lines:
i = 0
new_line = ""
for word in line:
empty = ""
nb = max_ln[i] - len(word)
for s in range(nb + 2):
empty += " "
new_line += word + empty
i += 1
print(new_line)
def sendmail(self, account, recipients, subject, body, attachments,
smtp_host, smtp_port, html):
"""
:param account: Account or nothing for anon.
:param recipients: str("*****@*****.**"), Account list of (Account
or string).
:param subject: str("My Subject").
:param body: str("My Body").
:param attachments: file or list of file.
:param smtp_host: str("localhost").
:param smtp_port: int(25).
:param html: allow html into mail body (booleen).
"""
charset.add_charset('utf-8', charset.SHORTEST, charset.QP)
# Verify account
account_firstname = account.firstname
account_lastname = account.lastname
account_mail = account.mail
if not account_mail:
self.logger.info(
'No mail adress for this user (Fill the mail account field)')
account_mail = '{0}@{1}'.format(account.user, socket.gethostname())
if isinstance(account_mail, (list, tuple)):
account_mail = account_mail[0]
if not account_lastname and not account_firstname:
account_full_mail = '"{0}" <{1}>'.format(
account_mail.split('@')[0].title(), account_mail)
else:
account_full_mail = account.get_full_mail()
if not re.match("^[a-zA-Z0-9._%-]+@[a-zA-Z0-9._%-]+.([a-zA-Z]{2,6})?$",
str(account_mail)):
return (
2, 'Invalid Email format for sender: {0}'.format(account_mail))
# Verify recipients
if not recipients:
return (2, 'No recipients configured')
if not isinstance(recipients, list):
recipients = [recipients]
dests = []
for dest in recipients:
if isinstance(dest, basestring):
if re.match(
"^[a-zA-Z0-9._%-]+@[a-zA-Z0-9._%-]+.([a-zA-Z]{2,6})?$",
dest):
dest_mail = dest
dest_full_mail = '"{0}" <{1}>'.format(
dest_mail.split('@')[0].title(), dest_mail)
dests.append(dest_full_mail)
else:
self.logger.error(
'Ignoring invalid recipient: {0}'.format(dest))
dests_str = ', '.join(dests)
# Verify attachments
if attachments:
storage = Storage(account=account, namespace='object')
if not isinstance(attachments, list):
attachments = [attachments]
# Send
msg = MIMEMultipart()
msg["From"] = account_full_mail
msg["To"] = dests_str
msg["Subject"] = subject
if html:
msg.attach(MIMEText(body, 'html', _charset='utf-8'))
else:
msg.attach(MIMEText(body, 'plain', _charset='utf-8'))
msg['Date'] = formatdate(localtime=True)
if attachments:
for _file in attachments:
part = MIMEBase('application', "octet-stream")
#meta_file = _file.get(storage)
content_file = _file.get(storage)
part.set_payload(content_file)
Encoders.encode_base64(part)
part.add_header(
'Content-Disposition',
'attachment; filename="%s"' % _file.data['file_name'])
part.add_header('Content-Type', _file.data['content_type'])
msg.attach(part)
sock = socket.socket()
try:
sock.connect((smtp_host, smtp_port))
except Exception as err:
return (2, 'Connection to SMTP <{0}:{1}> failed: {2}'.format(
smtp_host, smtp_port, err))
try:
server = smtplib.SMTP(smtp_host, smtp_port)
server.sendmail(account_full_mail, dests, msg.as_string())
server.quit()
except Exception as err:
return (2, "Impossible to send mail: {0}".format(err))
return (0, "Mail sent successfully")
timer.start()
storage.remove(records)
timer.stop()
remove_nb = len(records)
remove_speed = int(remove_nb / timer.elapsed)
print(
" + Insert Speed:", insert_speed, "records/s (%s records)" % insert_nb)
print(
" + Read Speed:", read_speed, "records/s (%s records)" % read_nb)
print(
" + Update Speed:", update_speed, "records/s (%s records)" % update_nb)
print(
" + Remove Speed:", remove_speed, "records/s (%s records)" % remove_nb)
namespace = "bench-" + str(randint(0, 1000))
account = Account()
storage = Storage(
account=account, namespace=namespace, logging_level=INFO)
timer = Timer(logging_level=INFO)
print("Bench with 'anonymous' account ...")
account = Account()
go(account, 5000)
print("Bench with 'root' account ...")
account = Account(user="******", group="root")
go(account, 5000)
storage.drop_namespace(namespace)
