def readJson(filename):
data = []
try:
with open(filename) as f:
data = json.load(f)
Logger.v('Reading:', filename)
except Exception as e:
Logger.e(e)
return data
def sink(self, data):
if data is None or len(data) == 0:
Logger.e(self.id(), "no data, abort")
return
cols = self.cols.split(',')
content = HtmlFactory.gethtml(self.title, cols, data)
title = "[%s]" % self.title
mail_to = [self.address]
#MailSender.sendMail2(mail_to, title, content)
MailSender.sendMail(mail_to, title, content)
def sink(self, data):
if data is None or len(data) == 0:
Logger.e(self.id(), "no data, abort")
return
cols = self.cols.split(',')
content = HtmlFactory.gethtml(self.title, cols, data)
title = "[%s]" % self.title
mail_to = [self.address]
#MailSender.sendMail2(mail_to, title, content)
MailSender.sendMail(mail_to, title, content)
def sink(self, data):
if data is None:
Logger.e(self.id(), "no data, abort")
return
cols = self.cols.split(',')
content = ""
if "rich" == self.html_format:
content = RichHtmlFactory.gethtml(self.title, cols, data)
elif "c3" == self.html_format:
content = C3ChartFactory.getHtml(self.title, cols, data)
else:
content = HtmlFactory.gethtml(self.title, cols, data)
return content
def sink(self, sections):
if sections is None:
Logger.e(self.id() + "no data, abort")
return
content = ""
for context in self.contexts:
key = context["id"]
section = sections[key]
content += "</p>"
content += HtmlFactory.getTable(context["title"], context['cols'].split(','), section)
content = HtmlFactory.getHeader(content)
return content
def sink(self, sections):
if sections is None:
Logger.e(self.id() + "no data, abort")
return
content = ""
for context in self.contexts:
key = context["id"]
section = sections[key]
content += "</p>"
content += HtmlFactory.getTable(context["title"],
context['cols'].split(','),
section)
content = HtmlFactory.getHeader(content)
return content
def sink(self, sections):
if sections is None or len(sections) == 0:
Logger.e(self.id() + "no data, abort")
return
content = ""
for context in self.contexts:
key = context["id"]
section = sections[key]
content += "</p>"
content += HtmlFactory.getTable(context["title"], context['cols'].split(','), section)
content = HtmlFactory.getHeader(content)
title = "[%s]" % self.title
mail_to = [self.address]
#MailSender.sendMail2(mail_to, title, content)
MailSender.sendMail(mail_to, title, content)
def process(conn, addr, dbManager):
SharedMemoryManager.setInstance(dbManager)
#set instance of shared memory to make all new thread sync into one memory at a time.
print('[%s][Start]' % addr[1])
#get and parse input params to json
data = conn.recv(4096)
if not data: #stop if no data
conn.close()
return
params = None
output = None
try:
args = data.decode('utf-8', 'ignore')
# basic.reloadlib(Router);
params, method = basic.decodeHeader(args)
# print('[%s][Process] %s'%(addr[1], params));
result = Router.route(params)
if (method == 'POST'):
header = 'POST HTTP/1.1 200 OK' if result[
'success'] else 'HTTP/1.1 500 Python Server Error'
output = fn.dumps("%s\n%s" % (header, str(result)))
else:
uncompress_data = fn.dumps(result, indent=0)
z = zlib.compressobj(-1, zlib.DEFLATED, 31)
gzip_compressed_data = z.compress(uncompress_data) + z.flush()
output = gzip_compressed_data
except Exception as ex:
Logger.e('Server:', ex)
traceback.print_exc()
result = {
'success': False,
'uncaught': ex
}
header = 'POST HTTP/1.1 200 OK' if result[
'success'] else 'HTTP/1.1 500 Python Server Error'
output = fn.dumps("%s\n%s" % (header, str()))
# if LIVE:
# Mail.send('[%s]%s:Error on server.py IP '%( DateTime.now(), basic.getCurrentIP()), '%s <br/> %s'%(params, ex));
send(conn, output)
conn.close()
print('[%s][Complete] ' % addr[1])
def sink(self, sections):
if sections is None or len(sections) == 0:
Logger.e(self.id() + "no data, abort")
return
for context in self.contexts:
key = context["id"]
output_file = context["output"]
cols = context["cols"].split(",")
write_header = context["write_header"]
section = sections[key]
with open(output_file, "w") as fp:
if write_header:
print >> fp, '\t'.join(cols)
for row in section:
rowdata = []
for item in row:
rowdata.append(str(item))
print >> fp, '\t'.join(rowdata)
def sink(self, sections):
if sections is None or len(sections) == 0:
Logger.e(self.id() + "no data, abort")
return
content = ""
for context in self.contexts:
key = context["id"]
section = sections[key]
content += "</p>"
content += HtmlFactory.getTable(context["title"],
context['cols'].split(','),
section)
content = HtmlFactory.getHeader(content)
title = "[%s]" % self.title
mail_to = [self.address]
#MailSender.sendMail2(mail_to, title, content)
MailSender.sendMail(mail_to, title, content)
def sink(self, data):
if data is None or len(data) == 0:
Logger.e(self.id() + "no data, abort")
return
sections = self.buildDataGroup(data)
keys = sections['__meta__']
content = ""
#for key in sections:
for key in keys:
section = sections[key]
content += "</p>"
content += HtmlFactory.getTable(section["title"], section['cols'].split(','), section['data'])
content = HtmlFactory.getHeader(content)
#print content
title = "[%s]" % self.title
mail_to = [self.address]
#MailSender.sendMail2(mail_to, title, content)
MailSender.sendMail(mail_to, title, content)
def sink(self, data):
if data is None or len(data) == 0:
Logger.e(self.id() + "no data, abort")
return
sections = self.buildDataGroup(data)
keys = sections['__meta__']
content = ""
#for key in sections:
for key in keys:
section = sections[key]
content += "</p>"
content += HtmlFactory.getTable(section["title"],
section['cols'].split(','),
section['data'])
content = HtmlFactory.getHeader(content)
#print content
title = "[%s]" % self.title
mail_to = [self.address]
#MailSender.sendMail2(mail_to, title, content)
MailSender.sendMail(mail_to, title, content)
class Executor:
def __init__(self, conf_file):
self.worker = Worker(conf_file)
def setUp(self):
pass
def run(self, task):
self.worker.run(task)
if __name__ == "__main__":
if len(sys.argv) <= 2:
Logger.e("[Main]", "no enough argument given, exit")
sys.exit(1)
conf_file = sys.argv[1]
task = sys.argv[2]
executor = Executor(conf_file)
executor.setUp()
executor.run(task)
class SimpleTestSupplier(BaseSupplier):
def get(self):
return [["val1", "val2"], ["val3", "val4"]]
def test():
def run(self, task):
Logger.d("[Executor-%s]" % task, "start running at " + str(datetime.datetime.now()))
if not self.config.has_section(task):
Logger.f("[Executor-%s]" % task, "failed running at {0}, reason={1}".format(datetime.datetime.now(), "task not found"))
return
supplier = None
supplier_type = self.config.get(task, "supplier.type").lower()
Logger.d("[Executor-%s]" % task, "supplier_type=%s" % (supplier_type))
if "sql" == supplier_type or "mysql" == supplier_type:
supplier_db_name = self.config.get(task, "supplier.db_name")
supplier_sql_file = self.config.get(task, "supplier.sql_file")
supplier_cachable = False
if self.config.has_option(task, "supplier.cachable"):
supplier_cachable = self.config.get(task, "supplier.cachable")
supplier_sql = ""
with open(supplier_sql_file, "r") as sql_file:
supplier_sql = sql_file.read()
#Logger.d("[Executor-%s]" % task, "db=%s file=%s sql=%s" % (supplier_db_name, supplier_sql_file, supplier_sql))
supplier = SQLBasedSupplier(task, supplier_db_name, supplier_sql, supplier_cachable)
elif "hql" == supplier_type or "hive" == supplier_type:
supplier_db_name = self.config.get(task, "supplier.db_name")
supplier_sql_file = self.config.get(task, "supplier.sql_file")
supplier_tmp_file = None
if self.config.has_option(task, "supplier.tmp_file"):
supplier_tmp_file = self.config.get(task, "supplier.tmp_file")
supplier_cachable = False
if self.config.has_option(task, "supplier.cachable"):
supplier_cachable = self.config.get(task, "supplier.cachable")
supplier_sql = ""
with open(supplier_sql_file, "r") as sql_file:
supplier_sql = sql_file.read()
#Logger.d("[Executor-%s]" % task, "db=%s file=%s sql=%s" % (supplier_db_name, supplier_sql_file, supplier_sql))
supplier = HiveBasedSupplier(task, supplier_db_name, supplier_sql, supplier_tmp_file, supplier_cachable)
elif "multi-sql" == supplier_type:
supplier_db_name = self.config.get(task, "supplier.db_name")
supplier_cachable = False
if self.config.has_option(task, "supplier.cachable"):
supplier_cachable = self.config.get(task, "supplier.cachable")
supplier_instances = self.config.get(task, "supplier.instances").split(",")
supplier_sqls = {}
for instance in supplier_instances:
instance_sql_file = self.config.get(task, "supplier." + instance + ".sql_file")
with open(instance_sql_file, "r") as sql_file:
instance_sql = sql_file.read()
supplier_sqls[instance] = {"sql":instance_sql}
supplier = MultiSQLBasedSupplier(task, supplier_db_name, supplier_sqls, supplier_cachable)
elif "http" == supplier_type:
supplier_product = self.config.get(task, "supplier.product")
supplier_cq = self.config.get(task, "supplier.cq")
supplier_dimen = self.config.get(task, "supplier.dimen")
supplier_log_date = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d")
if self.config.has_option(task, "supplier.date"):
supplier_date = self.config.get(task, "supplier.date")
if '{INTERVAL}' == supplier_date:
supplier_date_interval = self.config.get(task, "supplier.date.interval")
date_tmp = datetime.date.today() - datetime.timedelta(days=int(supplier_date_interval))
supplier_log_date = datetime.datetime.strftime(date_tmp, "%Y-%m-%d")
supplier_filters = {}
if self.config.has_option(task, "supplier.filters"):
supplier_filter_dims = self.config.get(task, "supplier.filters").split(",")
for dim in supplier_filter_dims:
dimValues = self.config.get(task, "supplier.filters." + dim).split(",")
supplier_filters[dim] = dimValues
supplier = HttpBasedSupplier(task, supplier_log_date, supplier_product, supplier_dimen, supplier_cq, supplier_filters)
elif "test" == supplier_type:
supplier = SimpleTestSupplier()
if supplier is None:
Logger.e("[Executor-%s]" % task, "no supplier, abort")
return
sinker = None
sinker_type = self.config.get(task, "sinker.type").lower()
if "html" == sinker_type:
sinker_title = self.config.get(task, "sinker.title")
sinker_cols = self.config.get(task, "sinker.cols")
sinker_html_format = self.config.get(task, "sinker.format")
sinker = HttpBasedSinker(task, sinker_title, sinker_cols, sinker_html_format)
elif "multi-html" == sinker_type:
sinker_title = self.config.get(task, "sinker.title")
sinker_instances = self.config.get(task, "sinker.instances").split(",")
sinker_contexts = []
for instance in sinker_instances:
instance_title = self.config.get(task, "sinker." + instance + ".title")
instance_cols = self.config.get(task, "sinker." + instance + ".cols")
sinker_contexts.append({"id":instance, "title":instance_title, "cols":instance_cols})
sinker = MultiSectionHttpSinker(task, sinker_title, sinker_contexts)
elif "email" == sinker_type:
sinker_title = self.config.get(task, "sinker.title")
sinker_cols = self.config.get(task, "sinker.cols")
sinker_address = self.config.get(task, "sinker.address")
sinker = EmailBasedSinker(task, sinker_address, sinker_title, sinker_cols)
elif "multi-email" == sinker_type:
sinker_address = self.config.get(task, "sinker.address")
sinker_title = self.config.get(task, "sinker.title")
sinker_instances = self.config.get(task, "sinker.instances").split(",")
sinker_contexts = []
for instance in sinker_instances:
instance_title = self.config.get(task, "sinker." + instance + ".title")
instance_cols = self.config.get(task, "sinker." + instance + ".cols")
sinker_contexts.append({"id":instance, "title":instance_title, "cols":instance_cols})
sinker = MultiSectionEmailSinker(task, sinker_address, sinker_title, sinker_contexts)
elif "multi-csv" == sinker_type:
sinker_instances = self.config.get(task, "sinker.instances").split(",")
sinker_contexts = []
for instance in sinker_instances:
instance_write_header = self.config.get(task, "sinker." + instance + ".write_header")
instance_cols = self.config.get(task, "sinker." + instance + ".cols")
instance_output = self.config.get(task, "sinker." + instance + ".output")
sinker_contexts.append({"id":instance, "write_header":instance_write_header, "cols":instance_cols, "output":instance_output})
sinker = MultiSectionCsvSinker(task, sinker_contexts)
elif "group-email" == sinker_type:
sinker_address = self.config.get(task, "sinker.address")
sinker_title = self.config.get(task, "sinker.title")
sinker_cols = self.config.get(task, "sinker.cols")
sinker_group_by = self.config.get(task, "sinker.group_by")
sinker_limit = sys.maxint
if self.config.has_option(task, "sinker.limit"):
sinker_limit = self.config.get(task, "sinker.limit")
sinker = GroupEmailSinker(task, sinker_address, sinker_title, sinker_cols, sinker_group_by, sinker_limit)
if sinker is None:
Logger.e("[Executor-%s]" % task, "no sinker, abort")
return
extender = None
extender_type = ""
if self.config.has_option(task, "extender.type"):
extender_type = self.config.get(task, "extender.type").lower()
if "single-key" == extender_type:
supplier_type = self.config.get(task, "extender.supplier.type")
extender_supplier = None
if "sql" == supplier_type or "mysql" == supplier_type:
supplier_db_name = self.config.get(task, "extender.supplier.db_name")
supplier_sql_file = self.config.get(task, "extender.supplier.sql_file")
supplier_cachable = False
if self.config.has_option(task, "extender.supplier.cachable"):
supplier_cachable = self.config.get(task, "extender.supplier.cachable")
supplier_sql = ""
with open(supplier_sql_file, "r") as sql_file:
supplier_sql = sql_file.read()
#Logger.d("[Executor-%s]" % task, "db=%s file=%s sql=%s" % (supplier_db_name, supplier_sql_file, supplier_sql))
extender_supplier = SQLBasedSupplier(task, supplier_db_name, supplier_sql, supplier_cachable)
if not extender_supplier is None:
extender_base_key_index = self.config.get(task, "extender.base_key_index")
extender_data_key_index = self.config.get(task, "extender.data_key_index")
extender_insert_index = self.config.get(task, "extender.insert_index")
extender = KeyBasedExtender(task, extender_supplier, extender_base_key_index, extender_data_key_index, extender_insert_index)
if "multi-stage" == extender_type:
extender_stages = self.config.get(task, "extender.stages").split(',')
contexts = []
for stage in extender_stages:
prefix = "extender." + stage + "."
supplier_type = self.config.get(task, prefix + "supplier.type")
extender_supplier = None
if "sql" == supplier_type or "mysql" == supplier_type:
supplier_db_name = self.config.get(task, prefix + "supplier.db_name")
supplier_sql_file = self.config.get(task, prefix + "supplier.sql_file")
supplier_cachable = False
if self.config.has_option(task, prefix + "supplier.cachable"):
supplier_cachable = self.config.get(task, prefix + "supplier.cachable")
supplier_sql = ""
with open(supplier_sql_file, "r") as sql_file:
supplier_sql = sql_file.read()
#Logger.d("[Executor-%s]" % task, "db=%s file=%s sql=%s" % (supplier_db_name, supplier_sql_file, supplier_sql))
extender_supplier = SQLBasedSupplier(task, supplier_db_name, supplier_sql, supplier_cachable)
if not extender_supplier is None:
extender_base_key_index = self.config.get(task, prefix + "base_key_index")
extender_data_key_index = self.config.get(task, prefix + "data_key_index")
extender_insert_index = self.config.get(task, prefix + "insert_index")
contexts.append({"supplier":extender_supplier, "base_key_index":extender_base_key_index, "data_key_index":extender_data_key_index, "insert_index":extender_insert_index})
extender = MultiStageKeyExtender(task, contexts)
data = supplier.get()
if not extender is None:
data = extender.extend(data)
data = sinker.sink(data)
Logger.d("[Executor-%s]" % task, "end running at " + str(datetime.datetime.now()))
return data
