def readJson(filename): data = [] try: with open(filename) as f: data = json.load(f) Logger.v('Reading:', filename) except Exception as e: Logger.e(e) return data
def sink(self, data): if data is None or len(data) == 0: Logger.e(self.id(), "no data, abort") return cols = self.cols.split(',') content = HtmlFactory.gethtml(self.title, cols, data) title = "[%s]" % self.title mail_to = [self.address] #MailSender.sendMail2(mail_to, title, content) MailSender.sendMail(mail_to, title, content)
def sink(self, data): if data is None: Logger.e(self.id(), "no data, abort") return cols = self.cols.split(',') content = "" if "rich" == self.html_format: content = RichHtmlFactory.gethtml(self.title, cols, data) elif "c3" == self.html_format: content = C3ChartFactory.getHtml(self.title, cols, data) else: content = HtmlFactory.gethtml(self.title, cols, data) return content
def sink(self, sections): if sections is None: Logger.e(self.id() + "no data, abort") return content = "" for context in self.contexts: key = context["id"] section = sections[key] content += "</p>" content += HtmlFactory.getTable(context["title"], context['cols'].split(','), section) content = HtmlFactory.getHeader(content) return content
def sink(self, sections): if sections is None or len(sections) == 0: Logger.e(self.id() + "no data, abort") return content = "" for context in self.contexts: key = context["id"] section = sections[key] content += "</p>" content += HtmlFactory.getTable(context["title"], context['cols'].split(','), section) content = HtmlFactory.getHeader(content) title = "[%s]" % self.title mail_to = [self.address] #MailSender.sendMail2(mail_to, title, content) MailSender.sendMail(mail_to, title, content)
def process(conn, addr, dbManager): SharedMemoryManager.setInstance(dbManager) #set instance of shared memory to make all new thread sync into one memory at a time. print('[%s][Start]' % addr[1]) #get and parse input params to json data = conn.recv(4096) if not data: #stop if no data conn.close() return params = None output = None try: args = data.decode('utf-8', 'ignore') # basic.reloadlib(Router); params, method = basic.decodeHeader(args) # print('[%s][Process] %s'%(addr[1], params)); result = Router.route(params) if (method == 'POST'): header = 'POST HTTP/1.1 200 OK' if result[ 'success'] else 'HTTP/1.1 500 Python Server Error' output = fn.dumps("%s\n%s" % (header, str(result))) else: uncompress_data = fn.dumps(result, indent=0) z = zlib.compressobj(-1, zlib.DEFLATED, 31) gzip_compressed_data = z.compress(uncompress_data) + z.flush() output = gzip_compressed_data except Exception as ex: Logger.e('Server:', ex) traceback.print_exc() result = { 'success': False, 'uncaught': ex } header = 'POST HTTP/1.1 200 OK' if result[ 'success'] else 'HTTP/1.1 500 Python Server Error' output = fn.dumps("%s\n%s" % (header, str())) # if LIVE: # Mail.send('[%s]%s:Error on server.py IP '%( DateTime.now(), basic.getCurrentIP()), '%s <br/> %s'%(params, ex)); send(conn, output) conn.close() print('[%s][Complete] ' % addr[1])
def sink(self, sections): if sections is None or len(sections) == 0: Logger.e(self.id() + "no data, abort") return for context in self.contexts: key = context["id"] output_file = context["output"] cols = context["cols"].split(",") write_header = context["write_header"] section = sections[key] with open(output_file, "w") as fp: if write_header: print >> fp, '\t'.join(cols) for row in section: rowdata = [] for item in row: rowdata.append(str(item)) print >> fp, '\t'.join(rowdata)
def sink(self, data): if data is None or len(data) == 0: Logger.e(self.id() + "no data, abort") return sections = self.buildDataGroup(data) keys = sections['__meta__'] content = "" #for key in sections: for key in keys: section = sections[key] content += "</p>" content += HtmlFactory.getTable(section["title"], section['cols'].split(','), section['data']) content = HtmlFactory.getHeader(content) #print content title = "[%s]" % self.title mail_to = [self.address] #MailSender.sendMail2(mail_to, title, content) MailSender.sendMail(mail_to, title, content)
class Executor: def __init__(self, conf_file): self.worker = Worker(conf_file) def setUp(self): pass def run(self, task): self.worker.run(task) if __name__ == "__main__": if len(sys.argv) <= 2: Logger.e("[Main]", "no enough argument given, exit") sys.exit(1) conf_file = sys.argv[1] task = sys.argv[2] executor = Executor(conf_file) executor.setUp() executor.run(task) class SimpleTestSupplier(BaseSupplier): def get(self): return [["val1", "val2"], ["val3", "val4"]] def test():
def run(self, task): Logger.d("[Executor-%s]" % task, "start running at " + str(datetime.datetime.now())) if not self.config.has_section(task): Logger.f("[Executor-%s]" % task, "failed running at {0}, reason={1}".format(datetime.datetime.now(), "task not found")) return supplier = None supplier_type = self.config.get(task, "supplier.type").lower() Logger.d("[Executor-%s]" % task, "supplier_type=%s" % (supplier_type)) if "sql" == supplier_type or "mysql" == supplier_type: supplier_db_name = self.config.get(task, "supplier.db_name") supplier_sql_file = self.config.get(task, "supplier.sql_file") supplier_cachable = False if self.config.has_option(task, "supplier.cachable"): supplier_cachable = self.config.get(task, "supplier.cachable") supplier_sql = "" with open(supplier_sql_file, "r") as sql_file: supplier_sql = sql_file.read() #Logger.d("[Executor-%s]" % task, "db=%s file=%s sql=%s" % (supplier_db_name, supplier_sql_file, supplier_sql)) supplier = SQLBasedSupplier(task, supplier_db_name, supplier_sql, supplier_cachable) elif "hql" == supplier_type or "hive" == supplier_type: supplier_db_name = self.config.get(task, "supplier.db_name") supplier_sql_file = self.config.get(task, "supplier.sql_file") supplier_tmp_file = None if self.config.has_option(task, "supplier.tmp_file"): supplier_tmp_file = self.config.get(task, "supplier.tmp_file") supplier_cachable = False if self.config.has_option(task, "supplier.cachable"): supplier_cachable = self.config.get(task, "supplier.cachable") supplier_sql = "" with open(supplier_sql_file, "r") as sql_file: supplier_sql = sql_file.read() #Logger.d("[Executor-%s]" % task, "db=%s file=%s sql=%s" % (supplier_db_name, supplier_sql_file, supplier_sql)) supplier = HiveBasedSupplier(task, supplier_db_name, supplier_sql, supplier_tmp_file, supplier_cachable) elif "multi-sql" == supplier_type: supplier_db_name = self.config.get(task, "supplier.db_name") supplier_cachable = False if self.config.has_option(task, "supplier.cachable"): supplier_cachable = self.config.get(task, "supplier.cachable") supplier_instances = self.config.get(task, "supplier.instances").split(",") supplier_sqls = {} for instance in supplier_instances: instance_sql_file = self.config.get(task, "supplier." + instance + ".sql_file") with open(instance_sql_file, "r") as sql_file: instance_sql = sql_file.read() supplier_sqls[instance] = {"sql":instance_sql} supplier = MultiSQLBasedSupplier(task, supplier_db_name, supplier_sqls, supplier_cachable) elif "http" == supplier_type: supplier_product = self.config.get(task, "supplier.product") supplier_cq = self.config.get(task, "supplier.cq") supplier_dimen = self.config.get(task, "supplier.dimen") supplier_log_date = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d") if self.config.has_option(task, "supplier.date"): supplier_date = self.config.get(task, "supplier.date") if '{INTERVAL}' == supplier_date: supplier_date_interval = self.config.get(task, "supplier.date.interval") date_tmp = datetime.date.today() - datetime.timedelta(days=int(supplier_date_interval)) supplier_log_date = datetime.datetime.strftime(date_tmp, "%Y-%m-%d") supplier_filters = {} if self.config.has_option(task, "supplier.filters"): supplier_filter_dims = self.config.get(task, "supplier.filters").split(",") for dim in supplier_filter_dims: dimValues = self.config.get(task, "supplier.filters." + dim).split(",") supplier_filters[dim] = dimValues supplier = HttpBasedSupplier(task, supplier_log_date, supplier_product, supplier_dimen, supplier_cq, supplier_filters) elif "test" == supplier_type: supplier = SimpleTestSupplier() if supplier is None: Logger.e("[Executor-%s]" % task, "no supplier, abort") return sinker = None sinker_type = self.config.get(task, "sinker.type").lower() if "html" == sinker_type: sinker_title = self.config.get(task, "sinker.title") sinker_cols = self.config.get(task, "sinker.cols") sinker_html_format = self.config.get(task, "sinker.format") sinker = HttpBasedSinker(task, sinker_title, sinker_cols, sinker_html_format) elif "multi-html" == sinker_type: sinker_title = self.config.get(task, "sinker.title") sinker_instances = self.config.get(task, "sinker.instances").split(",") sinker_contexts = [] for instance in sinker_instances: instance_title = self.config.get(task, "sinker." + instance + ".title") instance_cols = self.config.get(task, "sinker." + instance + ".cols") sinker_contexts.append({"id":instance, "title":instance_title, "cols":instance_cols}) sinker = MultiSectionHttpSinker(task, sinker_title, sinker_contexts) elif "email" == sinker_type: sinker_title = self.config.get(task, "sinker.title") sinker_cols = self.config.get(task, "sinker.cols") sinker_address = self.config.get(task, "sinker.address") sinker = EmailBasedSinker(task, sinker_address, sinker_title, sinker_cols) elif "multi-email" == sinker_type: sinker_address = self.config.get(task, "sinker.address") sinker_title = self.config.get(task, "sinker.title") sinker_instances = self.config.get(task, "sinker.instances").split(",") sinker_contexts = [] for instance in sinker_instances: instance_title = self.config.get(task, "sinker." + instance + ".title") instance_cols = self.config.get(task, "sinker." + instance + ".cols") sinker_contexts.append({"id":instance, "title":instance_title, "cols":instance_cols}) sinker = MultiSectionEmailSinker(task, sinker_address, sinker_title, sinker_contexts) elif "multi-csv" == sinker_type: sinker_instances = self.config.get(task, "sinker.instances").split(",") sinker_contexts = [] for instance in sinker_instances: instance_write_header = self.config.get(task, "sinker." + instance + ".write_header") instance_cols = self.config.get(task, "sinker." + instance + ".cols") instance_output = self.config.get(task, "sinker." + instance + ".output") sinker_contexts.append({"id":instance, "write_header":instance_write_header, "cols":instance_cols, "output":instance_output}) sinker = MultiSectionCsvSinker(task, sinker_contexts) elif "group-email" == sinker_type: sinker_address = self.config.get(task, "sinker.address") sinker_title = self.config.get(task, "sinker.title") sinker_cols = self.config.get(task, "sinker.cols") sinker_group_by = self.config.get(task, "sinker.group_by") sinker_limit = sys.maxint if self.config.has_option(task, "sinker.limit"): sinker_limit = self.config.get(task, "sinker.limit") sinker = GroupEmailSinker(task, sinker_address, sinker_title, sinker_cols, sinker_group_by, sinker_limit) if sinker is None: Logger.e("[Executor-%s]" % task, "no sinker, abort") return extender = None extender_type = "" if self.config.has_option(task, "extender.type"): extender_type = self.config.get(task, "extender.type").lower() if "single-key" == extender_type: supplier_type = self.config.get(task, "extender.supplier.type") extender_supplier = None if "sql" == supplier_type or "mysql" == supplier_type: supplier_db_name = self.config.get(task, "extender.supplier.db_name") supplier_sql_file = self.config.get(task, "extender.supplier.sql_file") supplier_cachable = False if self.config.has_option(task, "extender.supplier.cachable"): supplier_cachable = self.config.get(task, "extender.supplier.cachable") supplier_sql = "" with open(supplier_sql_file, "r") as sql_file: supplier_sql = sql_file.read() #Logger.d("[Executor-%s]" % task, "db=%s file=%s sql=%s" % (supplier_db_name, supplier_sql_file, supplier_sql)) extender_supplier = SQLBasedSupplier(task, supplier_db_name, supplier_sql, supplier_cachable) if not extender_supplier is None: extender_base_key_index = self.config.get(task, "extender.base_key_index") extender_data_key_index = self.config.get(task, "extender.data_key_index") extender_insert_index = self.config.get(task, "extender.insert_index") extender = KeyBasedExtender(task, extender_supplier, extender_base_key_index, extender_data_key_index, extender_insert_index) if "multi-stage" == extender_type: extender_stages = self.config.get(task, "extender.stages").split(',') contexts = [] for stage in extender_stages: prefix = "extender." + stage + "." supplier_type = self.config.get(task, prefix + "supplier.type") extender_supplier = None if "sql" == supplier_type or "mysql" == supplier_type: supplier_db_name = self.config.get(task, prefix + "supplier.db_name") supplier_sql_file = self.config.get(task, prefix + "supplier.sql_file") supplier_cachable = False if self.config.has_option(task, prefix + "supplier.cachable"): supplier_cachable = self.config.get(task, prefix + "supplier.cachable") supplier_sql = "" with open(supplier_sql_file, "r") as sql_file: supplier_sql = sql_file.read() #Logger.d("[Executor-%s]" % task, "db=%s file=%s sql=%s" % (supplier_db_name, supplier_sql_file, supplier_sql)) extender_supplier = SQLBasedSupplier(task, supplier_db_name, supplier_sql, supplier_cachable) if not extender_supplier is None: extender_base_key_index = self.config.get(task, prefix + "base_key_index") extender_data_key_index = self.config.get(task, prefix + "data_key_index") extender_insert_index = self.config.get(task, prefix + "insert_index") contexts.append({"supplier":extender_supplier, "base_key_index":extender_base_key_index, "data_key_index":extender_data_key_index, "insert_index":extender_insert_index}) extender = MultiStageKeyExtender(task, contexts) data = supplier.get() if not extender is None: data = extender.extend(data) data = sinker.sink(data) Logger.d("[Executor-%s]" % task, "end running at " + str(datetime.datetime.now())) return data