def initiate_usage_scheduler(self):
usage_scheduler_run_time = datetime.datetime.now()
if self.run_count != 0:
print 'usage_scheduler started at: ' + str(usage_scheduler_run_time)
logger.info('usage_scheduler started at: ' + str(usage_scheduler_run_time))
datasets = bq.get_datasets()
for dataset in datasets:
try:
storage_query = "SELECT SUM(size_bytes) as storage_bq FROM [{0}.__TABLES__]".format(
dataset['datasetReference']['datasetId'])
storage_bq = bq.execute_query(storage_query, user_id=0, tenant='DigInEngine')[0]['storage_bq']
user_id = db.get_data(
"SELECT user_id FROM digin_user_settings WHERE REPLACE(REPLACE(email, '.', '_'), '@', '_') = '{0}' limit 1".format(
dataset['datasetReference']['datasetId']))['rows']
if user_id == ():
print 'No user_Settings found for user: '******'datasetReference']['datasetId']
logger.info('No user_Settings found for user: '******'datasetReference']['datasetId'])
else:
usages = {'storage_bq': 0 if storage_bq is None else storage_bq}
obj = dre.RatingEngine(user_id[0][0], 'undefined', **usages)
obj.set_usage()
print 'Storage calculated for user ' + str(user_id[0][0])
logger.info('Storage calculated for user ' + str(user_id[0][0]))
except Exception, err:
print err
def _calculate_summary(self):
summary = db.get_data("SELECT parameter, value FROM digin_usage_summary "
"WHERE user_id = '{0}' AND tenant = '{1}'".format(self.user_id, self.tenant))
if summary['rows'] == ():
db.insert_data(self.insert_obj,'digin_usage_summary')
else:
update_obj = []
residue_insert = []
for i in self.insert_obj:
for j in summary['rows']:
if i['parameter'] == j[0]:
if self.is_increment:
update_obj.append({i['parameter']:str(int(j[1])+int(i['value']))})
else:
update_obj.append({i['parameter']: str(int(j[1])-int(i['value']))})
break
else:
residue_insert.append({'parameter':i['parameter'],'value':i['value'],'user_id':self.user_id,'tenant':self.tenant})
continue
if update_obj:
for record in update_obj:
db.update_data('digin_usage_summary',"WHERE parameter = '{0}' AND user_id = '{1}' AND tenant = '{2}' "
.format(list(record.keys())[0], self.user_id,self.tenant), value = record.itervalues().next(),
modifieddatetime=datetime.datetime.now())
if residue_insert:
db.insert_data(residue_insert,'digin_usage_summary')
def get_package_attributes(self):
package= self.package_id
query = "SELECT " \
"package_id, " \
"package_name, " \
"package_attribute, " \
"package_value, " \
"package_price " \
"FROM digin_packagedetails " \
"WHERE package_id = {0} ".format(self.package_id)
try:
result = db.get_data(query)['rows']
data_list = []
for row in result:
data = {
'package_name': row[1],
'package_attribute': row[2],
'package_value': row[3],
'package_price': row[4]}
data_list.append(data)
except Exception, err:
print err
return "Error occurred while getting data"
def get_package_detail(self):
query = "SELECT " \
"a.package_id, " \
"a.package_name, " \
"a.package_attribute, " \
"SUM(a.package_value), " \
"SUM(a.package_price), " \
"b.expiry_datetime, " \
"TIMESTAMPDIFF(DAY, CURRENT_TIMESTAMP, expiry_datetime) as remaining_days, " \
"CURRENT_TIMESTAMP > b.expiry_datetime " \
"FROM digin_packagedetails a " \
"INNER JOIN digin_tenant_package_details b " \
"ON a.package_id = b.package_id " \
"WHERE b.tenant_id = '{0}' AND b.package_status = 'current_package' " \
"GROUP BY a.package_id, a.package_name, a.package_attribute, b.expiry_datetime, remaining_days".format(self.tenant)
try:
result = db.get_data(query)['rows']
data_list = []
for row in result:
data = {'package_id': row[0],
'package_name': row[1],
'package_attribute': row[2],
'package_value_sum': row[3],
'package_price_sum': row[4],
'expiry_datetime': row[5],
'remaining_days': row[6],
'is_expired': bool(row[7])}
data_list.append(data)
except Exception, err:
print err
return cmg.format_response(False, err, "Error occurred while getting data", exception=sys.exc_info())
def get_user_settings(user_id, domain):
query = "SELECT components, user_role, cache_lifetime, widget_limit, " \
"query_limit, logo_path, dp_path, theme_config, modified_date_time, created_date_time, " \
"domain FROM digin_user_settings WHERE user_id = '{0}' AND domain = '{1}'".format(user_id, domain)
logo_path = conf.get_conf('FilePathConfig.ini','User Files')['Path']
document_root = conf.get_conf('FilePathConfig.ini','Document Root')['Path']
path = re.sub(document_root, '', logo_path)
try:
user_data = CC.get_data(query)
if user_data['rows'] == ():
logger.info('No user settings saved for given user ' + user_id)
return cmg.format_response(True,user_id,"No user settings saved for given user and domain")
data ={
'components': user_data['rows'][0][0],
'user_role': user_data['rows'][0][1],
'cache_lifetime': int(user_data['rows'][0][2]),
'widget_limit': int(user_data['rows'][0][3]),
'query_limit': int(user_data['rows'][0][4]),
'logo_path': path+user_data['rows'][0][5],
'dp_path': path+user_data['rows'][0][6],
'theme_config': user_data['rows'][0][7],
'modified_date_time': user_data['rows'][0][8],
'created_date_time': user_data['rows'][0][9],
'domain': user_data['rows'][0][10]
}
except Exception, err:
logger.error("Error retrieving user settings")
logger.error(err)
print "Error retrieving user settings"
print err
raise
def get_rating_summary(self):
user_dict = {}
if self.security_level == 'admin':
summary = db.get_data("SELECT user_id, parameter, SUM(value) as value FROM digin_usage_summary "
"WHERE tenant = '{0}' GROUP BY user_id, parameter".format(self.tenant))['rows']
else:
summary = db.get_data("SELECT user_id, parameter, value FROM digin_usage_summary "
"WHERE user_id = '{0}' AND tenant = '{1}'".format(self.user_id, self.tenant))['rows']
for parameter in summary:
if parameter[0] in user_dict:
user_dict[parameter[0]][parameter[1]]=parameter[2]
else:
user_dict[parameter[0]] = {parameter[1]:parameter[2]}
rated_dict = {'usage': [{self.tenant:user_dict}],
'exceed_blocked': euc.ExceedUsageCalculator(tenant=self.tenant, attribute=None).calculation()}
return cmg.format_response('True',rated_dict,"Usage data retrieved")
def wcloud_stream(tweet):
try:
logger.info('Getting data from Cache if there is')
c_data = json.loads(cache.get_data('wordcloud','d_name,data','d_name ="test1"').data)
logger.info('calc word count for a new tweet')
dat = wnc.wordcloud_json(tweet)
n = json.loads(dat)
logger.info('Agreegate previous and current tweets')
for t in n:
if t in c_data:
c_data[t] = c_data[t] + n[t]
else:
c_data.update({t:n[t]})
logger.info('update MemSql')
c_data=dict(sorted(c_data.items(), key=lambda x: x[1],reverse=True)[:250])
update_data(json.dumps(c_data))
except Exception, err:
logger.info(err, 'Insert to MemSQL, if dash boardname is ot existing')
dat = wnc.wordcloud_json(tweet)
c_data = json.loads(dat)
c_data=dict(sorted(c_data.items(), key=lambda x: x[1],reverse=True)[:250])
insert_data(json.dumps(c_data))
def ret_fuzzyC(dbtype, rec_data, u_id, cache_timeout):
time = datetime.datetime.now()
try:
cache_existance = CC.get_data("SELECT expirydatetime >= '{0}' FROM cache_algorithms "
"WHERE id = '{1}' and name_algo='fuzzyC'".format(time, u_id))['rows']
except Exception, err:
logger.error(err, "Error connecting to cache..")
cache_existance = ()
def get_usage(self):
if self.attribute == "data":
try:
summary = db.get_data(" SELECT sum(value) FROM digin_usage_summary "
" WHERE tenant = '{0}' AND parameter = 'download_bq' OR 'upload_size_bq' ".format(self.tenant))['rows']
return summary[0]
except Exception, err:
print "Error inserting to DB!"
return cmg.format_response(False, err, "Error occurred while getting packages details .. \n" + str(err),
exception=sys.exc_info())
def get_rating_detail(self, params):
date_dict = {}
if self.security_level == 'admin':
detail = db.get_data("SELECT user_id, DATE(createddatetime), parameter, SUM(value) as value FROM digin_usage_details "
"WHERE tenant = '{0}' AND DATE(createddatetime)>= {1} AND DATE(createddatetime)<= {2} "
"GROUP BY user_id, DATE(createddatetime), parameter".format(self.tenant, params.start_date, params.end_date))['rows']
else:
detail = db.get_data("SELECT user_id, DATE(createddatetime), parameter, SUM(value) as value FROM digin_usage_details "
"WHERE user_id = '{0}' AND tenant = '{1}' AND DATE(createddatetime)>= {2} AND DATE(createddatetime)<= {3} "
"GROUP BY user_id, DATE(createddatetime), parameter".format(self.user_id, self.tenant, params.start_date, params.end_date))['rows']
for row in detail:
if row[0] in date_dict:
if str(row[1]) in date_dict[row[0]]:
date_dict[row[0]][str(row[1])][row[2]] = row[3]
else:
date_dict[row[0]][str(row[1])] = {row[2]:row[3]}
else:
date_dict[row[0]] = {str(row[1]):{row[2]:row[3]}}
rated_detail = [{self.tenant:date_dict}]
return cmg.format_response('True',rated_detail,"Usage data retrieved")
def _temporary_delete_components(comp_id, table, user_id, domain):
try:
is_access_valid = CC.get_data("SELECT digin_comp_id FROM digin_component_header a INNER JOIN "
"digin_component_access_details b ON a.digin_comp_id = b.component_id "
"WHERE digin_comp_id = {0} AND domain = '{1}' AND user_id = '{2}'".format(comp_id,domain,user_id))
if int(is_access_valid['rows'][0][0]) == int(comp_id):
result = CC.update_data(table,'WHERE digin_comp_id = {0}'.format(comp_id),is_active=False)
_rate_calculation_helper(False, comp_id, 'dashboard', user_id, domain) # Component count is decremented when components moved to Trash
print 'Component deleted user_id: {0}, tenant: {1}, component_id: {2}'.format(user_id, domain, comp_id)
else:
result = 'No component found for deletion user_id: {0}, tenant: {1}, component_id: {2}'.format(user_id,domain,comp_id)
print result
return result
except Exception, err:
print err
raise
def get_ledger(self):
# query = "SELECT " \
# "a.package_id, " \
# "a.package_name, " \
# "a.package_attribute, " \
# "a.package_value, " \
# "a.package_price, " \
# "b.expiry_datetime, " \
# "TIMESTAMPDIFF(DAY, CURRENT_TIMESTAMP, expiry_datetime) as remaining_days, " \
# "b.package_status, " \
# "b.created_datetime " \
# "FROM digin_packagedetails a " \
# "INNER JOIN digin_tenant_package_details b " \
# "ON a.package_id = b.package_id " \
# "WHERE b.tenant_id = '{0}' " \
# "AND b.created_datetime >= TIMESTAMP('{1}') AND b.created_datetime <= TIMESTAMP('{2}') " \
# "GROUP BY a.package_id, a.package_name, a.package_attribute, b.expiry_datetime, remaining_days " \
# "ORDER BY b.created_datetime ".format(self.tenant, self.start_date, self.end_date)
query = "SELECT " \
"package_id, " \
"expiry_datetime, " \
"TIMESTAMPDIFF(DAY, CURRENT_TIMESTAMP, expiry_datetime) as remaining_days, " \
"package_status, " \
"created_datetime " \
"FROM digin_tenant_package_details " \
"WHERE tenant_id = '{0}' " \
"AND created_datetime >= TIMESTAMP('{1}') AND created_datetime <= TIMESTAMP('{2}') " \
"ORDER BY created_datetime ".format(self.tenant, self.start_date, self.end_date)
try:
result = db.get_data(query)['rows']
data_list = []
for row in result:
data = {'package_id': row[0],
'package_Details': PackageProcessor(package_name=None, package_attribute=None, package_value=None, package_price=None,is_default=False, tenant =self.tenant,package_id=int(row[0])).get_package_attributes(),
'expiry_datetime': row[1],
'remaining_days': row[2],
'package_status': row[3],
'created_datetime': row[4]}
data_list.append(data)
except Exception, err:
print err
return cmg.format_response(False, err, "Error occurred while getting data", exception=sys.exc_info())
def get_all_components(params, user_id, domain):
try:
data = CC.get_data("SELECT h.digin_comp_id, h.digin_comp_name, h.digin_comp_type "
"FROM digin_component_access_details a "
"INNER JOIN digin_component_header h "
"ON a.component_id = h.digin_comp_id "
"WHERE h.is_active = TRUE AND a.domain = '{0}' AND a.user_id = '{1}' "
"AND a.type IN ('dashboard','report')".format(domain, user_id))
print data["rows"]
comps = []
for comp in data["rows"]:
comp_dict ={}
comp_dict["compID"]= comp[0]
comp_dict["compName"]=comp[1]
comp_dict["compType"]=comp[2]
comps.append(comp_dict)
return cmg.format_response(True,comps,"Successful!")
except Exception, err:
logger.error("Error getting data from cache. %s" % err)
return cmg.format_response(False,0,"Error Occurred!", exception = sys.exc_info())
def get_package_summary(self):
query = "SELECT " \
"a.package_attribute, " \
"SUM(a.package_value), " \
"SUM(a.package_price) " \
"FROM digin_packagedetails a " \
"INNER JOIN digin_tenant_package_details b " \
"ON a.package_id = b.package_id " \
"WHERE b.tenant_id = '{0}' AND b.package_status = 'current_package' " \
"GROUP BY a.package_attribute".format(self.tenant)
try:
result = db.get_data(query)['rows']
data_list = []
for row in result:
data = {'package_attribute': row[0],
'package_value_sum': row[1],
'package_price_sum': row[2]}
data_list.append(data)
except Exception, err:
print err
return cmg.format_response(False, err, "Error occurred while getting data", exception=sys.exc_info())
def store_user_settings(params,user_id, domain):
data_object = [{'user_id': user_id,
'email': params['email'],
'components': params['components'],
'user_role': params['user_role'],
'cache_lifetime': 300 if params['cache_lifetime'] is None else int(params['cache_lifetime']),
'widget_limit': default_user_settings['widget_limit'] if params['widget_limit'] is None else int(params['widget_limit']),
'query_limit': default_user_settings['query_limit'] if params['query_limit'] is None else int(params['query_limit']),
'logo_path': '/digin_user_data/'+user_id+'/'+domain+'/logos/'+params['logo_name'],
'dp_path': '/digin_user_data/'+user_id+'/'+domain+'/DPs/'+params['dp_name'],
'theme_config': params['theme_config'],
'modified_date_time': datetime.datetime.now(),
'created_date_time': datetime.datetime.now(),
'domain': domain
}]
logger.info("Data received!")
logger.info(data_object)
existance = CC.get_data("SELECT user_id from digin_user_settings where user_id = '{0}' AND domain = '{1}'".format(user_id, domain))
if existance['rows'] != ():
try:
CC.update_data('digin_user_settings',"WHERE user_id='{0}'".format(user_id),
components=params['components'],
user_role=params['user_role'],
cache_lifetime=int(params['cache_lifetime']),
widget_limit=int(params['widget_limit']),
query_limit=int(params['query_limit']),
logo_path='/digin_user_data/'+user_id+'/'+domain+'/logos/'+params['logo_name'],
dp_path='/digin_user_data/'+user_id+'/'+domain+'/DPs/'+params['dp_name'],
theme_config=params['theme_config'],
modified_date_time=datetime.datetime.now())
return cmg.format_response(True,1,"User settings updated successfully")
except Exception, err:
logger.error("Error updating user settings")
logger.error(err)
print "Error updating user settings"
print err
raise
def get_component_by_comp_id(params, user_id, domain):
try:
data = CC.get_data("SELECT h.digin_comp_id, h.digin_comp_name, h.refresh_interval, h.digin_comp_class, "
"h.digin_comp_type, h.digin_comp_category, h.created_date_time, p.page_id, p.page_name, "
"p.page_data, d.widget_id, d.widget_name, d.widget_data,d.row,d.col,d.size_x,d.size_y "
"FROM "
"(SELECT component_id FROM digin_component_access_details WHERE "
"component_id = {0} AND domain = '{1}' AND user_id = '{2}') a "
"LEFT JOIN "
"(SELECT digin_comp_id, digin_comp_name, refresh_interval, digin_comp_class, "
"digin_comp_type, digin_comp_category, created_date_time "
"FROM digin_component_header "
"WHERE is_active = TRUE AND digin_comp_id = {0}) h "
"ON a.component_id = h.digin_comp_id "
"LEFT JOIN "
"(SELECT page_id, page_name, page_data, digin_comp_id FROM digin_component_page_detail "
"WHERE is_active = TRUE) p "
"ON h.digin_comp_id = p.digin_comp_id "
"LEFT JOIN "
"(SELECT widget_id, widget_name, widget_data,size_x,size_y,col,row, digin_comp_id, comp_page_id FROM digin_component_detail "
"WHERE is_active = TRUE) d "
"ON h.digin_comp_id = d.digin_comp_id AND p.page_id = d.comp_page_id "
"ORDER BY d.widget_id ASC".format(params.comp_id, domain, user_id))
if data['rows'] == ():
print "No dashboard saved for given ID!"
return cmg.format_response(True,None,"No dashboard saved for given ID!")
component = {}
component['compID'] = data['rows'][0][0]
component['compName'] = data['rows'][0][1]
component['compClass'] = data['rows'][0][3]
component['compType'] = data['rows'][0][4]
component['compCategory'] = data['rows'][0][5]
component['refreshInterval'] = data['rows'][0][2]
def json_encode_decimal(obj):
if isinstance(obj, decimal.Decimal):
return str(obj)
raise TypeError(repr(obj) + " is not JSON serializable")
widget_ids = set(d[10] for d in data['rows'])
widgets = []
#filter(None, widget_ids[0])
for widget_id in widget_ids:
if widget_id is None:
continue
for record in data['rows']:
if widget_id == record[10]:
widget = {
'pageID':record[7],
'widgetID' : record[10],
'sizeX':record[15],
'sizeY':record[16],
'row':record[13],
'col':record[14],
'widgetName':record[11],
'widgetData':json.loads(record[12]) if record[12] is not None else None
}
widgets.append(widget)
pages = []
page_ids = set( d[7] for d in data['rows'])
for page_id in page_ids:
for record in data['rows']:
if page_id == record[7]:
page ={
'pageID': record[7],
'pageName': record[8],
'pageData': json.loads(record[9]) if record[9] is not None else None,
'widgets': []
}
pages.append(page)
break
pages_sorted = sorted(pages, key=itemgetter('pageID'))
widgets_sorted = sorted(widgets, key=itemgetter('widgetID'))
for page in pages_sorted:
for widget_ in widgets_sorted:
if widget_['pageID'] == page['pageID']:
#widget_.pop("pageID", None)
page['widgets'].append(widget_)
component['pages'] = pages_sorted
return cmg.format_response(True,component,"Successful!")
except Exception, err:
logger.error("Error getting data from cache. %s" % err)
return cmg.format_response(False,0,"Error Occurred!", exception = sys.exc_info())
'type':'dashboard',
'domain':domain}
access_detail_obj.append(access_detail_comp)
result_ad = CC.insert_data(access_detail_obj,'digin_component_access_details')
print result_ad
except Exception, err:
print err
return cmg.format_response(False,err,"Error Occurred while giving default dashboard access",exception=sys.exc_info())
#TODO insert menuids to access_details based on user's package
if ast.literal_eval(default_sys_settings['signup_sample_reports']):
logger.info("User will be given the default report access!")
print "User will be given the default report access!"
query = "SELECT digin_comp_name " \
"FROM digin_component_header WHERE digin_comp_id in {0} ".format(tuple(ast.literal_eval(default_user_settings['components_reports'])))
print query
user_data = CC.get_data(query)
report_names = user_data['rows']
try:
access_detail_obj = []
prs.ReportInitialConfig.prptConfig(user_id,domain,report_names)
prs.ReportInitialConfig.ktrConfig(user_id,domain,report_names)
for reports in ast.literal_eval(default_user_settings['components_reports']):
access_detail_comp = {'user_id':user_id,
'component_id':reports,
'type':'report',
'domain':domain}
access_detail_obj.append(access_detail_comp)
result_ad = CC.insert_data(access_detail_obj,'digin_component_access_details')
print result_ad
except Exception, err:
def get_usage(self):
if self.attribute == "data":
try:
summary = db.get_data(" SELECT sum(value) FROM digin_usage_summary "
" WHERE tenant = '{0}' AND parameter = 'download_bq' OR 'upload_size_bq' ".format(self.tenant))['rows']
return summary[0]
except Exception, err:
print "Error inserting to DB!"
return cmg.format_response(False, err, "Error occurred while getting packages details .. \n" + str(err),
exception=sys.exc_info())
elif self.attribute == "storage":
try:
summary = db.get_data(" SELECT sum(value) FROM digin_usage_summary "
" WHERE tenant = '{0}' AND parameter = 'storage_bq' ".format(self.tenant))['rows']
return summary[0]
except Exception, err:
print "Error inserting to DB!"
return cmg.format_response(False, err, "Error occurred while getting packages details .. \n" + str(err),
exception=sys.exc_info())
elif self.attribute == "users":
try:
summary = db.get_data(" SELECT sum(value) FROM digin_usage_summary "
" WHERE tenant = '{0}' AND parameter = 'users' ".format(self.tenant))['rows']
return summary[0]
except Exception, err:
print "Error inserting to DB!"
return cmg.format_response(False, err, "Error occurred while getting packages details .. \n" + str(err),
exception=sys.exc_info())
df = ret_data(dbtype, rec_data)
print 'dbtype', dbtype
print 'rec_data', rec_data
try:
output = fc.FuzzyC_algo(df)
cache_data(output, u_id, cache_timeout, name_algo='fuzzyC')
result = cmg.format_response(True,output,'fuzzyC processed successfully!')
except Exception, err:
logger.error(err)
result = cmg.format_response(False,err,'fuzzyC Failed!', sys.exc_info())
finally:
return result
else:
logger.info("Getting fuzzyC data from Cache..")
result = ''
try:
data = json.loads(CC.get_data("SELECT data FROM cache_algorithms "
"WHERE id = '{0}' and name_algo='fuzzyC'".format(u_id))['rows'][0][0])
result = cmg.format_response(True,data,'Data successfully processed!')
logger.info("Data received from cache")
except:
logger.error("Error occurred while fetching data from Cache")
result = cmg.format_response(False,None,'Error occurred while getting data from cache!',sys.exc_info())
raise
finally:
return result
