def send_report():
account = get_account()
reportStorage = cstorage(account=account, namespace='files')
recipients = request.params.get('recipients', default=None)
_id = request.params.get('_id', default=None)
body = request.params.get('body', default=None)
subject = request.params.get('subject', default=None)
meta = reportStorage.get(_id)
meta.__class__ = cfile
mail = {
'account': account,
'attachments': meta,
'recipients': recipients,
'subject': subject,
'body': body,
}
try:
task = task_mail.send.delay(**mail)
output = task.get()
return {'success': True, 'total': '1', 'data': {'output': output}}
except Exception, err:
logger.error('Error when run subtask mail : %s' % err)
return {
'success': False,
'total': '1',
'data': {
'output': 'Mail sending failed'
}
}
def files(metaId=None):
if metaId:
account = get_account()
storage = cstorage(account=account, namespace=namespace)
logger.debug("Get file '%s'" % metaId)
meta = storage.get(metaId)
meta.__class__ = cfile
file_name = meta.data['file_name']
content_type = meta.data['content_type']
logger.debug(" + File name: %s" % file_name)
logger.debug(" + Content type: %s" % content_type)
report = meta.get(storage)
if report:
response.headers['Content-Disposition'] = 'attachment; filename="%s"' % file_name
response.headers['Content-Type'] = content_type
try:
return report
except Exception, err:
logger.error(err)
else:
logger.error('No report found in gridfs')
return HTTPError(404, " Not Found")
def send_report():
account = get_account()
reportStorage = cstorage(account=account, namespace='files')
recipients = request.params.get('recipients', default=None)
_id = request.params.get('_id', default=None)
body = request.params.get('body', default=None)
subject = request.params.get('subject', default=None)
meta = reportStorage.get(_id)
meta.__class__ = cfile
mail = {
'account':account,
'attachments': meta,
'recipients':recipients,
'subject':subject,
'body': body,
}
try:
task = task_mail.send.delay(**mail)
output = task.get()
return {'success':True,'total':'1','data':{'output':output}}
except Exception, err:
logger.error('Error when run subtask mail : %s' % err)
return {'success':False,'total':'1','data':{'output':'Mail sending failed'}}
def setUp(self):
self.anonymous_account = caccount()
self.root_account = caccount(user="******", group="root")
self.storage = cstorage(self.root_account, namespace='unittest')
self.data = {}
self.my_file = '/opt/canopsis/bin/pydoc'
self.my_data = 'JAMBON123'
def test_01_Init(self):
global STORAGE
STORAGE = cstorage(self.user_account,
namespace='unittest',
logging_level=logging.DEBUG)
records = STORAGE.find(account=self.root_account)
STORAGE.remove(records, account=self.root_account)
def __init__(self, prompt, account, namespace='object', crecord_type=None):
ccmd.__init__(self, prompt)
self.account = account
self.namespace = namespace
self.crecord_type = crecord_type
self.storage = cstorage(account,
namespace=namespace,
logging_level=logging.INFO)
def init():
storage = cstorage(account=root)
namespaces = ['perfdata', 'perfdata.fs.files', 'perfdata.fs.chunks']
for namespace in namespaces:
logger.info(" + Drop '%s' collection" % namespace)
storage.drop_namespace(namespace)
def init():
storage = cstorage(account=root)
namespaces = ['files', 'binaries.files', 'binaries.chunks']
for namespace in namespaces:
logger.info(" + Drop '%s' collection" % namespace)
storage.drop_namespace(namespace)
def delete_file(metaId):
account = get_account()
storage = cstorage(account=account, namespace=namespace)
try :
storage.remove(metaId,account=account)
except:
logger.error('Failed to remove file')
return HTTPError(500, "Failed to remove file")
def generate_report(startTime, stopTime,view_name,mail=None):
account = get_account()
storage = cstorage(account=account, namespace='object')
if(isinstance(mail,str)):
try:
mail = json.loads(mail)
except Exception, err:
logger.error('Error while transform string mail to object' % err)
def launch_celery_task(*args, **kwargs):
if kwargs.has_key('task') and kwargs.has_key('method'):
try:
amqp = camqp(logging_name="aps-amqp")
amqp.start()
timer_begin = int(time.time())
#----------Get task informations
task_name = kwargs['_scheduled']
celery_task_name = kwargs['task']
module = __import__(kwargs['task'])
exec "task = module.%s" % kwargs['method']
#-------------Clear arguments
methodargs = kwargs
del methodargs['task']
del methodargs['method']
del kwargs['_scheduled']
#-------------execute task
success = True
try:
result = task.delay(*args, **methodargs)
result.get()
result = result.result
if not result['success']:
raise Exception('Celery task failed')
except Exception, err:
success = False
aps_error = str(err)
logger.error(err)
#------------Get account and storage
try:
if isinstance(kwargs['account'], unicode) or isinstance(
kwargs['account'], str):
account = caccount(user=kwargs['account'])
else:
account = kwargs['account']
#logger.error(account)
logger.info('Caccount create from passed arguments')
except Exception, err:
logger.info('No account specified in the task')
account = caccount()
try:
storage = cstorage(account=account, namespace='object')
except Exception, err:
logger.info('Error while fecthing storages : %s' % err)
success = False
aps_error = str(err)
def launch_celery_task(*args,**kwargs):
if kwargs.has_key('task') and kwargs.has_key('method'):
try:
amqp = camqp(logging_name="aps-amqp")
amqp.start()
timer_begin = int(time.time())
#----------Get task informations
task_name = kwargs['_scheduled']
celery_task_name = kwargs['task']
module = __import__(kwargs['task'])
exec "task = module.%s" % kwargs['method']
#-------------Clear arguments
methodargs = kwargs
del methodargs['task']
del methodargs['method']
del kwargs['_scheduled']
#-------------execute task
success = True
try:
result = task.delay(*args,**methodargs)
result.get()
result = result.result
if not result['success']:
raise Exception('Celery task failed')
except Exception, err:
success = False
aps_error = str(err)
logger.error(err)
#------------Get account and storage
try:
if isinstance(kwargs['account'],unicode) or isinstance(kwargs['account'],str):
account = caccount(user=kwargs['account'])
else:
account = kwargs['account']
#logger.error(account)
logger.info('Caccount create from passed arguments')
except Exception, err:
logger.info('No account specified in the task')
account = caccount()
try:
storage = cstorage(account=account, namespace='object')
except Exception, err:
logger.info('Error while fecthing storages : %s' % err)
success = False
aps_error = str(err)
def generate_report(startTime, stopTime,view_name,mail=None):
account = get_account()
if not check_group_rights(account,group_managing_access):
return HTTPError(403, 'Insufficient rights')
storage = cstorage(account=account, namespace='object')
if(isinstance(mail,str)):
try:
mail = json.loads(mail)
except Exception, err:
logger.error('Error while transform string mail to object' % err)
def put_in_grid_fs(file_path, file_name, account):
storage = cstorage(account, namespace='files')
report = cfile(storage=storage)
report.put_file(file_path, file_name, content_type='application/pdf')
report.data['creationTs'] = int(time.time())
id = storage.put(report)
if not report.check(storage):
logger.error('Report not in grid fs')
return False
else:
return id
def get_cache(storage=None): global CACHE if not storage: storage = cstorage(caccount(), namespace='cache') if CACHE: return CACHE else: #global CACHE CACHE = ccache(storage, namespace='cache') return CACHE
def get_cache(storage=None):
global CACHE
if not storage:
storage = cstorage(caccount(), namespace='cache')
if CACHE:
return CACHE
else:
#global CACHE
CACHE = ccache(storage, namespace='cache')
return CACHE
def update_for_new_rights():
#update briefcase elements
storage = cstorage(namespace='files', account=root)
dump = storage.find({})
for record in dump:
if record.owner.find('account.') == -1:
record.owner = 'account.%s' % record.owner
if record.group.find('group.') == -1:
record.group = 'group.%s' % record.group
storage.put(dump)
def update_for_new_rights():
#update briefcase elements
storage = cstorage(namespace='files',account=root)
dump = storage.find({})
for record in dump:
if record.owner.find('account.') == -1:
record.owner = 'account.%s' % record.owner
if record.group.find('group.') == -1:
record.group = 'group.%s' % record.group
storage.put(dump)
def generate_report(startTime, stopTime,view_name,mail=None, timezone=time.timezone):
stopTime = int(stopTime)
startTime = int(startTime)
account = get_account()
storage = cstorage(account=account, namespace='object')
if mail:
try:
mail = json.loads(mail)
except Exception, err:
logger.error('Error while transform string mail to object' % err)
mail = None
def generate_report(startTime, stopTime, view_name, mail=None):
stopTime = int(stopTime)
startTime = int(startTime)
account = get_account()
storage = cstorage(account=account, namespace='object')
if mail:
try:
mail = json.loads(mail)
except Exception, err:
logger.error('Error while transform string mail to object' % err)
mail = None
def put_in_grid_fs(file_path, file_name, account,owner=None):
storage = cstorage(account, namespace='files')
report = cfile(storage=storage)
report.put_file(file_path, file_name, content_type='application/pdf')
if owner:
report.chown(owner)
id = storage.put(report)
if not report.check(storage):
logger.error('Report not in grid fs')
return False
else:
return id
def put_in_grid_fs(file_path, file_name, account,owner=None):
storage = cstorage(account, namespace='files')
report = cfile(storage=storage)
report.put_file(file_path, file_name, content_type='application/pdf')
if owner:
report.chown(owner)
id = storage.put(report)
if not report.check(storage):
logger.error('Report not in grid fs')
return False
else:
return id
def test_1_Init(self):
global myamqp
myamqp = camqp()
myamqp.add_queue(
queue_name="unittest_alerts", routing_keys="#", callback=on_alert, exchange_name=myamqp.exchange_name_alerts
)
myamqp.start()
time.sleep(1)
global storage
storage = cstorage(caccount(user="******", group="root"), namespace="events", logging_level=logging.DEBUG)
global perfstore
perfstore = pyperfstore2.manager(logging_level=logging.DEBUG)
clean()
def list_files():
limit = int(request.params.get('limit', default=20))
start = int(request.params.get('start', default=0))
sort = request.params.get('sort', default=None)
filter = request.params.get('filter', default={})
###########account and storage
account = get_account()
storage = cstorage(account=account, namespace=namespace)
logger.debug("List files")
###########load filter
if filter:
try:
filter = json.loads(filter)
except Exception, err:
logger.error("Filter decode: %s" % err)
filter = {}
def test_1_Init(self):
global myamqp
myamqp = camqp()
myamqp.add_queue(queue_name="unittest_alerts",
routing_keys="#",
callback=on_alert,
exchange_name=myamqp.exchange_name_alerts)
myamqp.start()
time.sleep(1)
global storage
storage = cstorage(caccount(user="******", group="root"),
namespace='events',
logging_level=logging.DEBUG)
global perfstore
perfstore = pyperfstore2.manager(logging_level=logging.DEBUG)
clean()
bdd_account = caccount(STORAGE.get(account._id))
bdd_group = cgroup(STORAGE.get(group._id))
if group._id not in bdd_account.groups:
raise Exception(
'Group corruption while stock in bdd after add in group')
if account._id not in bdd_group.account_ids:
raise Exception(
'Group corruption while stock in bdd after add in group')
'''
account.remove_from_groups(group._id)
bdd_account = caccount(STORAGE.get(bdd_account._id))
bdd_group = cgroup(STORAGE.get(bdd_group._id))
if group._id in bdd_account.groups:
raise Exception('Group corruption while stock in bdd after remove from group')
if account._id in bdd_group.account_ids:
raise Exception('Group corruption while stock in bdd after remove from group')
'''
def test_99_DropNamespace(self):
STORAGE.drop_namespace('unittest')
if __name__ == "__main__":
STORAGE = cstorage(caccount(user="******", group="root"),
namespace='unittest')
unittest.main(verbosity=2)
def render_pdf(fileName=None, viewName=None, startTime=None, stopTime=None, interval=None, account=None, mail=None, owner=None, orientation='Portrait', pagesize='A4'):
if not stopTime:
stopTime = int(time.time())
if not startTime:
if interval:
startTime = stopTime - interval
else:
startTime = -1
if viewName is None:
raise ValueError("task_render_pdf: you must at least provide a viewName")
#check if the account is just a name or a real caccount
if isinstance(account ,str) or isinstance(account ,unicode):
root_account = caccount(user='******',group='root',mail='*****@*****.**')
root_storage = cstorage(account=root_account, namespace='object')
bd_account = root_storage.find_one(mfilter={'_id':'account.%s' % str(account)})
if isinstance(bd_account, crecord):
account = caccount(bd_account)
logger.info('Successfuly retrieve right user from db')
else:
account = caccount(mail='*****@*****.**')
logger.info('Anonymous account created')
#get view options
storage = cstorage(account=account, namespace='object')
try:
view_record = storage.get(viewName,account=account)
except:
raise Exception("Impossible to find view '%s' with account '%s'" % (viewName, account._id))
logger.info("Account '%s' ask a rendering of view '%s' (%s)" % (account.name, view_record.name, viewName,))
#set fileName
if fileName is None:
toDate = date.fromtimestamp(int(stopTime))
if startTime and startTime != -1:
fromDate = date.fromtimestamp(int(startTime))
fileName = '%s_From_%s_To_%s.pdf' % (view_record.name, fromDate, toDate)
else:
fileName = '%s_%s.pdf' % (view_record.name,toDate)
logger.info('fileName: %s' % fileName)
ascii_fileName = hashlib.md5(fileName.encode('ascii', 'ignore')).hexdigest()
#get orientation and pagesize
view_options = view_record.data.get('view_options', {})
if isinstance(view_options, dict):
orientation = view_options.get('orientation', 'Portrait')
pagesize = view_options.get('pageSize', 'A4')
logger.info('Orientation: %s' % orientation)
logger.info('Pagesize: %s' % pagesize)
wrapper_conf_file = os.path.expanduser("~/etc/wkhtmltopdf_wrapper.json")
file_path = open(wrapper_conf_file, "r").read()
file_path = '%s/%s' % (json.loads(file_path)['report_dir'],ascii_fileName)
#create wrapper object
wkhtml_wrapper = Wrapper( ascii_fileName,
viewName,
startTime,
stopTime,
account,
wrapper_conf_file,
orientation=orientation,
pagesize=pagesize)
# Run rendering
logger.debug('Run pdf rendering')
wkhtml_wrapper.run_report()
logger.info('Put it in grid fs: %s' % file_path)
doc_id = put_in_grid_fs(file_path, fileName, account,owner)
logger.debug('Remove tmp report file with docId: %s' % doc_id)
os.remove(file_path)
#Subtask mail (if needed)
if isinstance(mail, dict):
#get cfile
try:
reportStorage = cstorage(account=account, namespace='files')
meta = reportStorage.get(doc_id)
meta.__class__ = cfile
except Exception, err:
logger.error('Error while fetching cfile : %s' % err)
try:
mail['account'] = account
mail['attachments'] = meta
result = task_mail.send.subtask(kwargs=mail).delay()
result.get()
result = result.result
#if subtask fail, raise exception
if(result['success'] == False):
raise Exception('Subtask mail have failed : %s' % result['celery_output'][0])
except Exception, err:
logger.error(err)
raise Exception('Impossible to send mail')
def render_pdf(
fileName=None,
viewName=None,
startTime=None,
stopTime=None,
interval=None,
account=None,
mail=None,
owner=None,
orientation="Portrait",
pagesize="A4",
):
if not stopTime:
stopTime = int(time.time())
if not startTime:
if interval:
startTime = stopTime - interval
else:
startTime = -1
if viewName is None:
raise ValueError("task_render_pdf: you must at least provide a viewName")
# check if the account is just a name or a real caccount
if isinstance(account, str) or isinstance(account, unicode):
root_account = caccount(user="******", group="root", mail="*****@*****.**")
root_storage = cstorage(account=root_account, namespace="object")
bd_account = root_storage.find_one(mfilter={"_id": "account.%s" % str(account)})
if isinstance(bd_account, crecord):
account = caccount(bd_account)
logger.info("Successfuly retrieve right user from db")
else:
account = caccount(mail="*****@*****.**")
logger.info("Anonymous account created")
# get view options
storage = cstorage(account=account, namespace="object")
try:
view_record = storage.get(viewName, account=account)
except:
raise Exception("Impossible to find view '%s' with account '%s'" % (viewName, account._id))
# set fileName
if fileName is None:
toDate = date.fromtimestamp(int(stopTime))
if startTime and startTime != -1:
fromDate = date.fromtimestamp(int(startTime))
fileName = "%s_From_%s_To_%s.pdf" % (view_record.name, fromDate, toDate)
else:
fileName = "%s_%s.pdf" % (view_record.name, toDate)
logger.info("fileName: %s" % fileName)
ascii_fileName = hashlib.md5(fileName.encode("ascii", "ignore")).hexdigest()
# get orientation and pagesize
view_options = view_record.data.get("view_options", {})
if isinstance(view_options, dict):
orientation = view_options.get("orientation", "Portrait")
pagesize = view_options.get("pageSize", "A4")
logger.info("Orientation: %s" % orientation)
logger.info("Pagesize: %s" % pagesize)
wrapper_conf_file = os.path.expanduser("~/etc/wkhtmltopdf_wrapper.json")
file_path = open(wrapper_conf_file, "r").read()
file_path = "%s/%s" % (json.loads(file_path)["report_dir"], ascii_fileName)
# create wrapper object
wkhtml_wrapper = Wrapper(
ascii_fileName,
viewName,
startTime,
stopTime,
account,
wrapper_conf_file,
orientation=orientation,
pagesize=pagesize,
)
# Run rendering
logger.debug("Run pdf rendering")
wkhtml_wrapper.run_report()
logger.info("Put it in grid fs: %s" % file_path)
doc_id = put_in_grid_fs(file_path, fileName, account, owner)
logger.debug("Remove tmp report file with docId: %s" % doc_id)
os.remove(file_path)
# Subtask mail (if needed)
if isinstance(mail, dict):
# get cfile
try:
reportStorage = cstorage(account=account, namespace="files")
meta = reportStorage.get(doc_id)
meta.__class__ = cfile
except Exception, err:
logger.error("Error while fetching cfile : %s" % err)
try:
mail["account"] = account
mail["attachments"] = meta
result = task_mail.send.subtask(kwargs=mail).delay()
result.get()
result = result.result
# if subtask fail, raise exception
if result["success"] == False:
raise Exception("Subtask mail have failed : %s" % result["celery_output"][0])
except Exception, err:
logger.error(err)
raise Exception("Impossible to send mail")
def send(account=None,
recipients=None,
subject=None,
body=None,
attachments=None,
smtp_host="localhost",
smtp_port=25,
html=False):
"""
account : caccount or nothing for anon
recipients : str("*****@*****.**"), caccount
list of (caccount or string)
subject : str("My Subject")
body : str("My Body")
attachments : cfile, list of cfile
smtp_host : str("localhost")
smtp_port : int(25)
html : allow html into mail body (booleen)
"""
###########
# Account #
###########
# Defaults
account_firstname = ""
account_lastname = ""
account_mail = ""
account_full_mail = ""
if isinstance(account, caccount):
account_firstname = account.firstname
account_lastname = account.lastname
account_mail = account.mail
if not account_mail:
logger.info(
'No mail adress for this user (Fill the mail account field)')
account_mail = '%s@%s' % (account.user, socket.gethostname())
if isinstance(account_mail, (list, tuple)):
account_mail = account_mail[0]
if not account_lastname and not account_firstname:
account_full_mail = "\"%s\" <%s>" % (
account_mail.split('@')[0].title(), account_mail)
else:
account_full_mail = account.get_full_mail()
if not re.match("^[a-zA-Z0-9._%-]+@[a-zA-Z0-9._%-]+.([a-zA-Z]{2,6})?$",
str(account_mail)):
raise ValueError('Invalid Email format for sender')
else:
raise Exception('Need caccount object in account')
##############
# Recipients #
##############
if not recipients:
raise ValueError('Give at least one recipient')
if not isinstance(recipients, list):
recipients = [recipients]
dests = []
for dest in recipients:
if isinstance(dest, caccount):
dest_firstname = dest.firstname
dest_lastname = dest.lastname
dest_mail = dest.mail
dest_full_mail = dest.get_full_mail()
dests.append(dest_full_mail)
elif isinstance(dest, str) or isinstance(dest, unicode):
if re.match("^[a-zA-Z0-9._%-]+@[a-zA-Z0-9._%-]+.([a-zA-Z]{2,6})?$",
dest):
dest_mail = dest
dest_full_mail = "\"%s\" <%s>" % (
dest_mail.split('@')[0].title(), dest_mail)
dests.append(dest_full_mail)
else:
raise ValueError('Invalid Email format for recipients')
else:
raise ValueError('Invalid Email format for recipients')
dests_str = ', '.join(dests)
###############
# Attachments #
###############
if attachments:
storage = cstorage(account=account, namespace='object')
if not isinstance(attachments, list):
attachments = [attachments]
########
# Send #
########
logger.debug('-----')
logger.debug('From: %s' % account_full_mail)
logger.debug('To : %s' % dests_str)
logger.debug('-----')
logger.debug('Subject: %s' % subject)
logger.debug('Body : %s' % body)
logger.debug('Attach.: %s' % attachments)
logger.debug('-----')
msg = MIMEMultipart()
msg["From"] = account_full_mail
msg["To"] = dests_str
msg["Subject"] = subject
if html:
msg.attach(MIMEText(body, 'html'))
else:
msg.attach(MIMEText(body, 'plain'))
msg['Date'] = formatdate(localtime=True)
if attachments:
for _file in attachments:
part = MIMEBase('application', "octet-stream")
if not isinstance(_file, cfile):
_file.__class__ = cfile
#meta_file = _file.get(storage)
content_file = _file.get(storage)
part.set_payload(content_file)
Encoders.encode_base64(part)
part.add_header(
'Content-Disposition',
'attachment; filename="%s"' % _file.data['file_name'])
part.add_header('Content-Type', _file.data['content_type'])
msg.attach(part)
s = socket.socket()
try:
s.connect((smtp_host, smtp_port))
except Exception as err:
raise Exception('something\'s wrong with %s:%d. Exception type is %s' %
(smtp_host, smtp_port, err))
try:
server = smtplib.SMTP(smtp_host, smtp_port)
server.sendmail(account_full_mail, dests, msg.as_string())
server.quit()
except Exception, err:
return "Error: unable to send email (%s)" % err
parser.add_argument('-fnam', '--first-name')
parser.add_argument('-lnam', '--last-name')
parser.add_argument('-debg', '--debug-output')
if len(sys.argv)==1:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
if (args.debug_output):
verbosity = logging.DEBUG
else:
verbosity = logging.ERROR
whoami = caccount(user="******", group="root")
my_storage = cstorage(whoami, namespace='object', logging_level=verbosity)
def user_exist(cstorage, user_name):
try:
my_user = caccount_get(cstorage,user_name)
except:
my_user = None
if (args.debug_output):
print ("DEBUG: my_user = %s \n" %my_user)
return my_user
if (args.list_users):
all_users = caccount_getall(my_storage)
counter = 0
for each_user in all_users:
counter +=1
def send(
account=None,
recipients=None,
subject=None,
body=None,
attachments=None,
smtp_host="localhost",
smtp_port=25,
html=False,
):
"""
account : caccount or nothing for anon
recipients : str("*****@*****.**"), caccount
list of (caccount or string)
subject : str("My Subject")
body : str("My Body")
attachments : cfile, list of cfile
smtp_host : str("localhost")
smtp_port : int(25)
html : allow html into mail body (booleen)
"""
###########
# Account #
###########
# Defaults
account_firstname = ""
account_lastname = ""
account_mail = ""
account_full_mail = ""
if isinstance(account, caccount):
account_firstname = account.firstname
account_lastname = account.lastname
account_mail = account.mail
if not account_mail:
logger.warning("No mail adress (Fill the mail field)")
account_mail = "%s@%s" % (account.user, socket.gethostname())
if not account_lastname and not account_firstname:
account_full_mail = '"%s" <%s>' % (account_mail.split("@")[0].title(), account_mail)
else:
account_full_mail = account.get_full_mail()
if not re.match("^[a-zA-Z0-9._%-]+@[a-zA-Z0-9._%-]+.([a-zA-Z]{2,6})?$", str(account_mail)):
raise ValueError("Invalid Email format for sender")
else:
raise Exception("Need caccount object in account")
##############
# Recipients #
##############
if not recipients:
raise ValueError("Give at least one recipient")
if not isinstance(recipients, list):
recipients = [recipients]
dests = []
for dest in recipients:
if isinstance(dest, caccount):
dest_firstname = dest.firstname
dest_lastname = dest.lastname
dest_mail = dest.mail
dest_full_mail = dest.get_full_mail()
dests.append(dest_full_mail)
elif isinstance(dest, str) or isinstance(dest, unicode):
if re.match("^[a-zA-Z0-9._%-]+@[a-zA-Z0-9._%-]+.([a-zA-Z]{2,6})?$", dest):
dest_mail = dest
dest_full_mail = '"%s" <%s>' % (dest_mail.split("@")[0].title(), dest_mail)
dests.append(dest_full_mail)
else:
raise ValueError("Invalid Email format for recipients")
else:
raise ValueError("Invalid Email format for recipients")
dests_str = ", ".join(dests)
###############
# Attachments #
###############
if attachments:
storage = cstorage(account=account, namespace="object")
if not isinstance(attachments, list):
attachments = [attachments]
########
# Send #
########
logger.debug("-----")
logger.debug("From: %s" % account_full_mail)
logger.debug("To : %s" % dests_str)
logger.debug("-----")
logger.debug("Subject: %s" % subject)
logger.debug("Body : %s" % body)
logger.debug("Attach.: %s" % attachments)
logger.debug("-----")
msg = MIMEMultipart()
msg["From"] = account_full_mail
msg["To"] = dests_str
msg["Subject"] = subject
if html:
msg.attach(MIMEText(body, "html"))
else:
msg.attach(MIMEText(body, "plain"))
msg["Date"] = formatdate(localtime=True)
if attachments:
for file in attachments:
part = MIMEBase("application", "octet-stream")
if not isinstance(file, cfile):
file.__class__ = cfile
meta_file = file.get(storage)
content_file = meta_file.read()
part.set_payload(content_file)
Encoders.encode_base64(part)
part.add_header("Content-Disposition", 'attachment; filename="%s"' % meta_file.name)
part.add_header("Content-Type", meta_file.content_type)
msg.attach(part)
s = socket.socket()
try:
s.connect((smtp_host, smtp_port))
except Exception, e:
raise Exception("something's wrong with %s:%d. Exception type is %s" % (smtp_host, smtp_port, ` e `))
# Canopsis is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Canopsis. If not, see <http://www.gnu.org/licenses/>.
# ---------------------------------
from caccount import caccount
from cstorage import cstorage
from crecord import crecord
##set root account
root = caccount(user="******", group="root")
storage = cstorage(account=root)
logger = None
def init():
namespaces = ['cache', 'events', 'events_log', 'object' ]
for namespace in namespaces:
logger.info(" + Drop '%s' collection" % namespace)
storage.drop_namespace(namespace)
#logger.info(" + Create 'cache' collection")
## Create 100MB cache
#storage.db.create_collection('cache', options={'capped': True, 'size': 104857600})
def update():
account.add_in_groups(group._id)
bdd_account = caccount(STORAGE.get(account._id))
bdd_group = cgroup(STORAGE.get(group._id))
if group._id not in bdd_account.groups:
raise Exception('Group corruption while stock in bdd after add in group')
if account._id not in bdd_group.account_ids:
raise Exception('Group corruption while stock in bdd after add in group')
'''
account.remove_from_groups(group._id)
bdd_account = caccount(STORAGE.get(bdd_account._id))
bdd_group = cgroup(STORAGE.get(bdd_group._id))
if group._id in bdd_account.groups:
raise Exception('Group corruption while stock in bdd after remove from group')
if account._id in bdd_group.account_ids:
raise Exception('Group corruption while stock in bdd after remove from group')
'''
def test_99_DropNamespace(self):
STORAGE.drop_namespace('unittest')
if __name__ == "__main__":
STORAGE = cstorage(caccount(user="******", group="root"), namespace='unittest')
unittest.main(verbosity=2)
def test_01_Init(self): global STORAGE STORAGE = cstorage(self.user_account, namespace='unittest', logging_level=logging.DEBUG) records = STORAGE.find(account=self.root_account) STORAGE.remove(records, account=self.root_account)
except Exception, err:
logger.error('Not enough arguments: %s' % err)
return HTTPError(404, "Not enough arguments")
logger.debug(" + metaId: %s" % metaId)
logger.debug(" + file name: %s" % file_name)
if not metaId:
logger.error('No report Id specified')
return HTTPError(405, " No report Id specified")
if file_name:
###########account and storage
account = get_account()
storage = cstorage(account=account, namespace=namespace)
try:
document = storage.get(metaId)
if document:
document.data['file_name'] = file_name
document.name = file_name
storage.put(document)
except Exception, err:
logger.error("Error when updating report %s: %s" % (metaId,err))
return HTTPError(500, "Failed to update report")
@delete('/files/:metaId',apply=[check_auth])
def delete_file(metaId):
account = get_account()
def send(account=None, recipients=None, subject=None, body=None, attachments=None, smtp_host="localhost", smtp_port=25, html=False):
"""
account : caccount or nothing for anon
recipients : str("*****@*****.**"), caccount
list of (caccount or string)
subject : str("My Subject")
body : str("My Body")
attachments : cfile, list of cfile
smtp_host : str("localhost")
smtp_port : int(25)
html : allow html into mail body (booleen)
"""
###########
# Account #
###########
# Defaults
account_firstname = ""
account_lastname = ""
account_mail = ""
account_full_mail = ""
if isinstance(account, caccount):
account_firstname = account.firstname
account_lastname = account.lastname
account_mail = account.mail
if not account_mail:
logger.info('No mail adress for this user (Fill the mail account field)')
account_mail = '%s@%s' % (account.user,socket.gethostname())
if isinstance(account_mail, (list, tuple)):
account_mail = account_mail[0]
if not account_lastname and not account_firstname:
account_full_mail = "\"%s\" <%s>" % (account_mail.split('@')[0].title(), account_mail)
else:
account_full_mail = account.get_full_mail()
if not re.match("^[a-zA-Z0-9._%-]+@[a-zA-Z0-9._%-]+.([a-zA-Z]{2,6})?$", str(account_mail)):
raise ValueError('Invalid Email format for sender')
else:
raise Exception('Need caccount object in account')
##############
# Recipients #
##############
if not recipients:
raise ValueError('Give at least one recipient')
if not isinstance(recipients, list):
recipients = [recipients]
dests = []
for dest in recipients:
if isinstance(dest, caccount):
dest_firstname = dest.firstname
dest_lastname = dest.lastname
dest_mail = dest.mail
dest_full_mail = dest.get_full_mail()
dests.append(dest_full_mail)
elif isinstance(dest, str) or isinstance(dest, unicode):
if re.match("^[a-zA-Z0-9._%-]+@[a-zA-Z0-9._%-]+.([a-zA-Z]{2,6})?$", dest):
dest_mail = dest
dest_full_mail = "\"%s\" <%s>" % (dest_mail.split('@')[0].title(), dest_mail)
dests.append(dest_full_mail)
else:
raise ValueError('Invalid Email format for recipients')
else:
raise ValueError('Invalid Email format for recipients')
dests_str = ', '.join(dests)
###############
# Attachments #
###############
if attachments:
storage = cstorage(account=account, namespace='object')
if not isinstance(attachments, list):
attachments = [attachments]
########
# Send #
########
logger.debug('-----')
logger.debug('From: %s' % account_full_mail)
logger.debug('To : %s' % dests_str)
logger.debug('-----')
logger.debug('Subject: %s' % subject)
logger.debug('Body : %s' % body)
logger.debug('Attach.: %s' % attachments)
logger.debug('-----')
msg = MIMEMultipart()
msg["From"] = account_full_mail
msg["To"] = dests_str
msg["Subject"] = subject
if html:
msg.attach(MIMEText(body, 'html'))
else:
msg.attach(MIMEText(body, 'plain'))
msg['Date'] = formatdate(localtime=True)
if attachments:
for _file in attachments:
part = MIMEBase('application', "octet-stream")
if not isinstance(_file, cfile):
_file.__class__ = cfile
#meta_file = _file.get(storage)
content_file = _file.get(storage)
part.set_payload(content_file)
Encoders.encode_base64(part)
part.add_header('Content-Disposition', 'attachment; filename="%s"' % _file.data['file_name'])
part.add_header('Content-Type', _file.data['content_type'])
msg.attach(part)
s = socket.socket()
try:
s.connect((smtp_host, smtp_port))
except Exception as err:
raise Exception('something\'s wrong with %s:%d. Exception type is %s' % (smtp_host, smtp_port, err))
try:
server = smtplib.SMTP(smtp_host, smtp_port)
server.sendmail(account_full_mail, dests, msg.as_string())
server.quit()
except Exception, err:
return "Error: unable to send email (%s)" % err
parser.add_argument('-fnam', '--first-name')
parser.add_argument('-lnam', '--last-name')
parser.add_argument('-debg', '--debug-output')
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
if (args.debug_output):
verbosity = logging.DEBUG
else:
verbosity = logging.ERROR
whoami = caccount(user="******", group="root")
my_storage = cstorage(whoami, namespace='object', logging_level=verbosity)
def user_exist(cstorage, user_name):
try:
my_user = caccount_get(cstorage, user_name)
except:
my_user = None
if (args.debug_output):
print("DEBUG: my_user = %s \n" % my_user)
return my_user
if (args.list_users):
all_users = caccount_getall(my_storage)
counter = 0
def render_pdf(filename=None, viewname=None, starttime=None, stoptime=None, account=None, wrapper_conf_file=None, mail=None):
if viewname is None:
raise ValueError("task_render_pdf : you must at least provide a viewname")
################setting variable for celery auto launch task##################"
#if no wrapper conf file set, take the default
if wrapper_conf_file is None:
wrapper_conf_file = "/opt/canopsis/etc/wkhtmltopdf_wrapper.json"
#check if the account is just a name or a real caccount
if isinstance(account ,str) or isinstance(account ,unicode):
root_account = caccount(user='******',group='root',mail='*****@*****.**')
root_storage = cstorage(account=root_account, namespace='object')
bd_account = root_storage.find_one(mfilter={'_id':'account.%s' % str(account)})
if isinstance(bd_account, crecord):
account = caccount(bd_account)
logger.info('Successfuly retrieve right user from db')
else:
account = caccount(mail='*****@*****.**')
logger.info('Anonymous account created')
#set stop time
if stoptime is None:
starttime = (time.time() - starttime) * 1000
stoptime = (time.time()) * 1000
#set filename
if filename is None:
fromDate = str(date.fromtimestamp(int(starttime) / 1000))
toDate = str(date.fromtimestamp(int(stoptime) / 1000))
#get crecord name of the view (id is really harsh)
storage = cstorage(account=account, namespace='object')
try:
record = storage.get(viewname,account=account)
crecord_name = record.name
except:
crecord_name = viewname
filename = '%s_From_%s_To_%s' % (crecord_name,fromDate,toDate) + '.pdf'
##############################################################################
libwkhtml_dir=os.path.expanduser("~/lib")
sys.path.append(libwkhtml_dir)
try:
import wkhtmltopdf.wrapper
# Generate config
settings = wkhtmltopdf.wrapper.load_conf( filename,
viewname,
starttime,
stoptime,
account,
wrapper_conf_file)
file_path = open(wrapper_conf_file, "r").read()
file_path = json.loads(file_path)['report_dir'] + "/" + filename
# Run rendering
logger.debug('Run pdf rendering')
result = wkhtmltopdf.wrapper.run(settings)
result.wait()
logger.debug('Put it in grid fs')
id = put_in_grid_fs(file_path, filename, account)
logger.debug('Remove tmp report file')
os.remove(file_path)
#Subtask mail (if needed)
if isinstance(mail, dict):
#get cfile
try:
reportStorage = cstorage(account=account, namespace='files')
meta = reportStorage.get(id)
meta.__class__ = cfile
except Exception, err:
logger.error('Error while fetching cfile : %s' % err)
try:
mail['account'] = account
mail['attachments'] = meta
result = task_mail.send.subtask(kwargs=mail).delay()
result.get()
result = result.result
#if subtask fail, raise exception
if(result['success'] == False):
raise Exception, 'Subtask mail have failed : %s' % result['celery_output'][0]
except Exception, err:
raise
def render_pdf(fileName=None, viewName=None, startTime=None, stopTime=None, interval=None, account=None, mail=None, owner=None, orientation='Portrait', pagesize='A4'):
if not stopTime:
stopTime = int(time.time())
if not startTime:
if interval:
startTime = stopTime - interval
else:
startTime = -1
if viewName is None:
raise ValueError("task_render_pdf: you must at least provide a viewName")
#check if the account is just a name or a real caccount
if isinstance(account ,str) or isinstance(account ,unicode):
root_account = caccount(user='******',group='root',mail='*****@*****.**')
root_storage = cstorage(account=root_account, namespace='object')
bd_account = root_storage.find_one(mfilter={'_id':'account.%s' % str(account)})
if isinstance(bd_account, crecord):
account = caccount(bd_account)
logger.info('Successfuly retrieve right user from db')
else:
account = caccount(mail='*****@*****.**')
logger.info('Anonymous account created')
#get view options
storage = cstorage(account=account, namespace='object')
try:
view_record = storage.get(viewName,account=account)
except:
raise Exception("Impossible to find view '%s' with account '%s'" % (viewName, account._id))
#set fileName
if fileName is None:
toDate = date.fromtimestamp(int(stopTime))
if startTime and startTime != -1:
fromDate = date.fromtimestamp(int(startTime))
fileName = '%s_From_%s_To_%s.pdf' % (view_record.name, fromDate, toDate)
else:
fileName = '%s_%s.pdf' % (view_record.name,toDate)
logger.info('fileName: %s' % fileName)
ascii_fileName = hashlib.md5(fileName.encode('ascii', 'ignore')).hexdigest()
#get orientation and pagesize
view_options = view_record.data.get('view_options', {})
if isinstance(view_options, dict):
orientation = view_options.get('orientation', 'Portrait')
pagesize = view_options.get('pageSize', 'A4')
logger.info('Orientation: %s' % orientation)
logger.info('Pagesize: %s' % pagesize)
wrapper_conf_file = os.path.expanduser("~/etc/wkhtmltopdf_wrapper.json")
file_path = open(wrapper_conf_file, "r").read()
file_path = '%s/%s' % (json.loads(file_path)['report_dir'],ascii_fileName)
#create wrapper object
wkhtml_wrapper = Wrapper( ascii_fileName,
viewName,
startTime,
stopTime,
account,
wrapper_conf_file,
orientation=orientation,
pagesize=pagesize)
# Run rendering
logger.debug('Run pdf rendering')
wkhtml_wrapper.run_report()
logger.info('Put it in grid fs: %s' % file_path)
doc_id = put_in_grid_fs(file_path, fileName, account,owner)
logger.debug('Remove tmp report file with docId: %s' % doc_id)
os.remove(file_path)
#Subtask mail (if needed)
if isinstance(mail, dict):
#get cfile
try:
reportStorage = cstorage(account=account, namespace='files')
meta = reportStorage.get(doc_id)
meta.__class__ = cfile
except Exception, err:
logger.error('Error while fetching cfile : %s' % err)
try:
mail['account'] = account
mail['attachments'] = meta
result = task_mail.send.subtask(kwargs=mail).delay()
result.get()
result = result.result
#if subtask fail, raise exception
if(result['success'] == False):
raise Exception('Subtask mail have failed : %s' % result['celery_output'][0])
except Exception, err:
logger.error(err)
raise Exception('Impossible to send mail')
def render_pdf(fileName=None, viewName=None, exporting=None, account=None, mail=None, owner=None, orientation='Portrait', pagesize='A4'):
logger.info('start render')
logger.debug("fileName: %s " % fileName)
logger.debug("viewName: %s " % viewName)
logger.debug("exporting: %s " % exporting)
logger.debug("account: %s " % account)
logger.debug("mail: %s " % mail)
if exporting is None:
exporting = {"enable": False}
now = time.time()
startTime = None
stopTime = None
logger.debug("now: %s " % now)
timezone = exporting.get(
'timezone',
{"type": "local", "value": 0})
if timezone.get('type') == 'utc':
timezone['value'] = 0
elif timezone.get('type') == 'server':
timezone['value'] = time.timezone
elif 'value' not in timezone:
timezone['value'] = 0
logger.error('timezone value does not exist')
if not exporting.get('enable', False):
stopTime = now
elif exporting.get('type', 'duration') == 'duration':
_datetime = datetime.fromtimestamp(now)
kwargs = {
exporting.get('unit', 'days'): int(exporting.get('length', 1))
}
rd = relativedelta(**kwargs)
_datetime -= rd
startTime = time.mktime(_datetime.timetuple())
stopTime = now
elif exporting['type'] == 'fixed':
logger.debug('now: %s' % now)
def get_timestamp(_time):
"""
Get a timestamp from an input _time struct.
"""
result = 0
_datetime = datetime.utcfromtimestamp(now)
before = _time.get('before')
if before is not None:
for key in before:
before[key] = int(before[key])
rd = relativedelta(**before)
_datetime -= rd
kwargs = dict()
day = _time.get('day')
if day is not None:
logger.debug("day: %s" % day)
kwargs['day'] = int(day)
month = _time.get('month')
if month is not None:
logger.debug("month: %s" % month)
kwargs['month'] = int(month)
hour = _time.get('hour')
if hour is not None:
logger.debug("hour: %s" % hour)
kwargs['hour'] = int(hour)
minute = _time.get('minute')
if minute is not None:
logger.debug("minute: %s" % minute)
kwargs['minute'] = int(minute)
logger.debug("_datetime: %s" % _datetime)
_datetime = _datetime.replace(**kwargs)
logger.debug("_datetime: %s" % _datetime)
day_of_week = _time.get('day_of_week')
if day_of_week is not None:
logger.debug("day_of_week: %s" % day_of_week)
day_of_week = int(day_of_week)
weekday = calendar.weekday(_datetime.year, _datetime.month, _datetime.day)
days = weekday - day_of_week
logger.debug("days %s" % days)
td = timedelta(days=days)
_datetime -= td
result = time.mktime(_datetime.utctimetuple())
result -= timezone['value']
logger.debug('result: %s' % result)
return result
startTime = None
_from = exporting.get('from')
if _from is not None and _from:
startTime = int(_from.get('timestamp', get_timestamp(_from)))
logger.info('from : {0} and startTime : {1} ({2})'.format(_from, startTime, datetime.utcfromtimestamp(startTime)))
stopTime = now
_to = exporting.get('to')
if startTime and _to is not None and _to.get('enable', False):
stopTime = int(_to.get('timestamp', get_timestamp(_to)))
logger.info('to : {0} and stopTime : {1} ({2})'.format(_to, stopTime, datetime.utcfromtimestamp(stopTime)))
else:
logger.error('Wrong exporting type %s' % exporting['type'])
if viewName is None:
raise ValueError("task_render_pdf: you must at least provide a viewName")
#check if the account is just a name or a real caccount
if isinstance(account ,str) or isinstance(account ,unicode):
root_account = caccount(user='******',group='root',mail='*****@*****.**')
root_storage = cstorage(account=root_account, namespace='object')
bd_account = root_storage.find_one(mfilter={'_id':'account.%s' % str(account)})
if isinstance(bd_account, crecord):
account = caccount(bd_account)
logger.info('Successfuly retrieve right user from db')
else:
account = caccount(mail='*****@*****.**')
logger.info('Anonymous account created')
#get view options
storage = cstorage(account=account, namespace='object')
try:
view_record = storage.get(viewName,account=account)
except:
raise Exception("Impossible to find view '%s' with account '%s'" % (viewName, account._id))
logger.info("Account '%s' ask a rendering of view '%s' (%s)" % (account.name, view_record.name, viewName,))
timezone_td = timedelta(
seconds=int(timezone.get('value', 0)))
def get_datetime_with_timezone(timestamp):
result = datetime.utcfromtimestamp(timestamp)
result -= timezone_td
return result
#set fileName
if fileName is None:
toDate = get_datetime_with_timezone(int(stopTime))
if startTime and startTime != -1:
fromDate = get_datetime_with_timezone(int(startTime))
fileName = '%s_From_%s_To_%s' % (view_record.name, fromDate, toDate)
else:
fileName = '%s_%s' % (view_record.name,toDate)
fileName += '_Tz_%s_%s' % (timezone.get('type', 'server'), (timezone.get('value', 0) if timezone.get('value', 0) >= 0 \
else 'minus_%s' % -timezone['value']))
fileName += '.pdf'
logger.info('fileName: %s' % fileName)
ascii_fileName = hashlib.md5(fileName.encode('ascii', 'ignore')).hexdigest()
#get orientation and pagesize
view_options = view_record.data.get('view_options', {})
if isinstance(view_options, dict):
orientation = view_options.get('orientation', 'Portrait')
pagesize = view_options.get('pageSize', 'A4')
logger.info('Orientation: %s' % orientation)
logger.info('Pagesize: %s' % pagesize)
wrapper_conf_file = os.path.expanduser("~/etc/wkhtmltopdf_wrapper.json")
file_path = open(wrapper_conf_file, "r").read()
file_path = '%s/%s' % (json.loads(file_path)['report_dir'],ascii_fileName)
#create wrapper object
wkhtml_wrapper = Wrapper( ascii_fileName,
viewName,
startTime,
stopTime,
account,
wrapper_conf_file,
orientation=orientation,
pagesize=pagesize)
wkhtml_wrapper.logger = logger
# Run rendering
logger.info('Run pdf rendering')
wkhtml_wrapper.run_report()
logger.info('Put it in grid fs: %s' % file_path)
try:
doc_id = put_in_grid_fs(file_path, fileName, account,owner)
except Exception as e:
import inspect
inspect.trace()
logger.info('Error in generating file (try to not use slink): %s, %s ' % (e, inspect.trace()))
logger.info('Remove tmp report file with docId: %s' % doc_id)
os.remove(file_path)
#Subtask mail (if needed)
if isinstance(mail, dict):
#get cfile
try:
reportStorage = cstorage(account=account, namespace='files')
meta = reportStorage.get(doc_id)
meta.__class__ = cfile
except Exception, err:
logger.error('Error while fetching cfile : %s' % err)
try:
mail['account'] = account
mail['attachments'] = meta
result = task_mail.send.subtask(kwargs=mail).delay()
result.get()
result = result.result
#if subtask fail, raise exception
if(result['success'] == False):
raise Exception('Subtask mail have failed : %s' % result['celery_output'][0])
except Exception, err:
logger.error(err)
raise Exception('Impossible to send mail')
update_speed = int(update_nb / timer.elapsed) ## Remove all records timer.start() storage.remove(records) timer.stop() remove_nb = len(records) remove_speed = int(remove_nb / timer.elapsed) print " + Insert Speed:",insert_speed,"records/s (%s records)" % insert_nb print " + Read Speed:",read_speed,"records/s (%s records)" % read_nb print " + Update Speed:",update_speed,"records/s (%s records)" % update_nb print " + Remove Speed:",remove_speed,"records/s (%s records)" % remove_nb namespace = "bench-"+str(random.randint(0,1000)) account = caccount() storage = cstorage(account=account, namespace=namespace, logging_level=logging.INFO) timer = ctimer(logging_level=logging.INFO) print "Bench with 'anonymous' account ..." account = caccount() go(account, 5000) print "Bench with 'root' account ..." account = caccount(user="******", group="root") go(account, 5000) storage.drop_namespace(namespace)
## Remove all records
timer.start()
storage.remove(records)
timer.stop()
remove_nb = len(records)
remove_speed = int(remove_nb / timer.elapsed)
print " + Insert Speed:", insert_speed, "records/s (%s records)" % insert_nb
print " + Read Speed:", read_speed, "records/s (%s records)" % read_nb
print " + Update Speed:", update_speed, "records/s (%s records)" % update_nb
print " + Remove Speed:", remove_speed, "records/s (%s records)" % remove_nb
namespace = "bench-" + str(random.randint(0, 1000))
account = caccount()
storage = cstorage(account=account,
namespace=namespace,
logging_level=logging.INFO)
timer = ctimer(logging_level=logging.INFO)
print "Bench with 'anonymous' account ..."
account = caccount()
go(account, 5000)
print "Bench with 'root' account ..."
account = caccount(user="******", group="root")
go(account, 5000)
storage.drop_namespace(namespace)
