def __init__(self, args): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) username = args[Constant.WH_DB_USERNAME_KEY] password = args[Constant.WH_DB_PASSWORD_KEY] JDBC_DRIVER = args[Constant.WH_DB_DRIVER_KEY] JDBC_URL = args[Constant.WH_DB_URL_KEY] self.input_table_file = args[Constant.ORA_SCHEMA_OUTPUT_KEY] self.input_field_file = args[Constant.ORA_FIELD_OUTPUT_KEY] self.input_sample_file = args[Constant.ORA_SAMPLE_OUTPUT_KEY] self.db_id = args[Constant.DB_ID_KEY] self.wh_etl_exec_id = args[Constant.WH_EXEC_ID_KEY] self.conn_mysql = zxJDBC.connect(JDBC_URL, username, password, JDBC_DRIVER) self.conn_cursor = self.conn_mysql.cursor() if Constant.INNODB_LOCK_WAIT_TIMEOUT in args: lock_wait_time = args[Constant.INNODB_LOCK_WAIT_TIMEOUT] self.conn_cursor.execute("SET innodb_lock_wait_timeout = %s;" % lock_wait_time) self.logger.info("Load Oracle Metadata into {}, db_id {}, wh_exec_id {}" .format(JDBC_URL, self.db_id, self.wh_etl_exec_id)) self.dict_dataset_table = 'dict_dataset' self.field_comments_table = 'field_comments' self.dict_field_table = 'dict_field_detail' self.dict_field_comment_table = 'dict_dataset_field_comment' self.dict_dataset_sample_table = 'dict_dataset_sample'
def __init__(self, args): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) username = args[Constant.WH_DB_USERNAME_KEY] password = args[Constant.WH_DB_PASSWORD_KEY] JDBC_DRIVER = args[Constant.WH_DB_DRIVER_KEY] JDBC_URL = args[Constant.WH_DB_URL_KEY] self.db_id = args[Constant.JOB_REF_ID_KEY] self.wh_etl_exec_id = args[Constant.WH_EXEC_ID_KEY] self.conn_mysql = zxJDBC.connect(JDBC_URL, username, password, JDBC_DRIVER) self.conn_cursor = self.conn_mysql.cursor() if Constant.INNODB_LOCK_WAIT_TIMEOUT in args: lock_wait_time = args[Constant.INNODB_LOCK_WAIT_TIMEOUT] self.conn_cursor.execute("SET innodb_lock_wait_timeout = %s;" % lock_wait_time) temp_dir = FileUtil.etl_temp_dir(args, "ORACLE") self.input_table_file = os.path.join(temp_dir, args[Constant.ORA_SCHEMA_OUTPUT_KEY]) self.input_field_file = os.path.join(temp_dir, args[Constant.ORA_FIELD_OUTPUT_KEY]) self.input_sample_file = os.path.join(temp_dir, args[Constant.ORA_SAMPLE_OUTPUT_KEY]) self.collect_sample = False if Constant.ORA_LOAD_SAMPLE in args: self.collect_sample = FileUtil.parse_bool(args[Constant.ORA_LOAD_SAMPLE], False) self.logger.info("Load Oracle Metadata into {}, db_id {}, wh_exec_id {}" .format(JDBC_URL, self.db_id, self.wh_etl_exec_id))
def __init__(self, hdfs_uri, kerberos=False, kerberos_principal=None, keytab_file=None): """ :param hdfs_uri: hdfs://hadoop-name-node:port :param kerberos: optional, if kerberos authentication is needed :param kerberos_principal: optional, [email protected] :param keytab_file: optional, absolute path to keytab file """ self.logger = LoggerFactory.getLogger(self.__class__.__name__) self.logger.info("keytab_file: " + keytab_file) hdfs_conf = Configuration() if hdfs_uri.startswith('hdfs://'): hdfs_conf.set(Hdfs.FS_DEFAULT_NAME_KEY, hdfs_uri) elif hdfs_uri > "": self.logger.error("%s is an invalid uri for hdfs namenode ipc bind." % hdfs_uri) if kerberos: # init kerberos and keytab if not kerberos_principal or not keytab_file or kerberos_principal == '' or keytab_file == '': print "Kerberos Principal and Keytab File Name/Path are required!" hdfs_conf.set("hadoop.security.authentication", "kerberos") hdfs_conf.set("dfs.namenode.kerberos.principal.pattern", "*") UserGroupInformation.setConfiguration(hdfs_conf) UserGroupInformation.loginUserFromKeytab(kerberos_principal, keytab_file) self.fs = Hdfs.get(hdfs_conf) requests.packages.urllib3.disable_warnings() self.logger.info("Initiated SchemaUrlHelper")
def __init__(self, args): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) self.app_id = int(args[Constant.JOB_REF_ID_KEY]) self.wh_exec_id = long(args[Constant.WH_EXEC_ID_KEY]) self.wh_con = zxJDBC.connect(args[Constant.WH_DB_URL_KEY], args[Constant.WH_DB_USERNAME_KEY], args[Constant.WH_DB_PASSWORD_KEY], args[Constant.WH_DB_DRIVER_KEY]) self.wh_cursor = self.wh_con.cursor() self.aw_con = self.get_connection(args[Constant.AW_DB_URL_KEY], args[Constant.AW_DB_PORT_KEY], args[Constant.AW_DB_NAME_KEY], args[Constant.AW_DB_USERNAME_KEY], args[Constant.AW_DB_PASSWORD_KEY], args[Constant.AW_DB_DRIVER_KEY]) self.aw_cursor = self.aw_con.cursor() self.lookback_period = args[Constant.AW_EXEC_ETL_LOOKBACK_KEY] self.app_folder = args[Constant.WH_APP_FOLDER_KEY] self.metadata_folder = self.app_folder + "/" + str(SchedulerType.APPWORX) + "/" + str(self.app_id) self.last_execution_unix_time = None self.get_last_execution_unix_time() if not os.path.exists(self.metadata_folder): try: os.makedirs(self.metadata_folder) except Exception as e: self.logger.error(e)
def setLogLevel(level): """setLogLevel(level) changes the log level to level, where level is a string (ERROR, WARN, INFO, DEBUG, and TRACE) """ logger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME) levels = {"ERROR": Level.ERROR, "WARN": Level.WARN, "INFO": Level.INFO, "DEBUG": Level.DEBUG, "TRACE": Level.TRACE} l = levels[level.upper()] logger.setLevel(l)
def __init__(self, args): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) self.wh_con = zxJDBC.connect(args[Constant.WH_DB_URL_KEY], args[Constant.WH_DB_USERNAME_KEY], args[Constant.WH_DB_PASSWORD_KEY], args[Constant.WH_DB_DRIVER_KEY]) self.wh_cursor = self.wh_con.cursor() self.app_id = int(args[Constant.JOB_REF_ID_KEY])
def __init__(self, name='max', labelList=('maxx','maxy','maxval', 'sum'), keyxlabel='maxx', keyylabel='maxy', formatString='Maximum value found to be at %f,%f (maxx,maxy) was %f (maxval). Sum was %f (sum)' ): self.logger = LoggerFactory.getLogger("SumMaxPositionAndValue:%s" % name) TwodDataSetProcessor.__init__(self, name, labelList, keyxlabel, keyylabel, formatString)
def __init__(self, args): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) self.wh_con = zxJDBC.connect(args[Constant.WH_DB_URL_KEY], args[Constant.WH_DB_USERNAME_KEY], args[Constant.WH_DB_PASSWORD_KEY], args[Constant.WH_DB_DRIVER_KEY]) self.wh_cursor = self.wh_con.cursor() self.wh_exec_id = long(args[Constant.WH_EXEC_ID_KEY]) self.app_folder = args[Constant.WH_APP_FOLDER_KEY]
def __init__(self, args): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) self.elasticsearch_index_url = args[Constant.WH_ELASTICSEARCH_URL_KEY] self.elasticsearch_port = args[Constant.WH_ELASTICSEARCH_PORT_KEY] self.wh_con = zxJDBC.connect(args[Constant.WH_DB_URL_KEY], args[Constant.WH_DB_USERNAME_KEY], args[Constant.WH_DB_PASSWORD_KEY], args[Constant.WH_DB_DRIVER_KEY]) self.wh_cursor = self.wh_con.cursor(1)
def __init__(self): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) username = args[Constant.HIVE_METASTORE_USERNAME] password = args[Constant.HIVE_METASTORE_PASSWORD] jdbc_driver = args[Constant.HIVE_METASTORE_JDBC_DRIVER] jdbc_url = args[Constant.HIVE_METASTORE_JDBC_URL] self.conn_hms = zxJDBC.connect(jdbc_url, username, password, jdbc_driver) self.curs = self.conn_hms.cursor() dependency_instance_file = args[Constant.HIVE_DEPENDENCY_CSV_FILE_KEY] self.instance_writer = FileWriter(dependency_instance_file)
def __init__(self, args, scheduler_type): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) self.app_id = int(args[Constant.APP_ID_KEY]) self.wh_con = zxJDBC.connect(args[Constant.WH_DB_URL_KEY], args[Constant.WH_DB_USERNAME_KEY], args[Constant.WH_DB_PASSWORD_KEY], args[Constant.WH_DB_DRIVER_KEY]) self.wh_cursor = self.wh_con.cursor() self.app_folder = args[Constant.WH_APP_FOLDER_KEY] self.metadata_folder = self.app_folder + "/" + str(scheduler_type) + "/" + str(self.app_id)
def __init__(self, path, iFileLoader=None, fileLoadTimout=None, printNfsTimes=False, wait_for_exposure_callable=None): self.logger = LoggerFactory.getLogger("LazyDataSetProvider:%s" % path) self.path = path self.iFileLoader = iFileLoader self.fileLoadTimout = fileLoadTimout self.printNfsTimes = printNfsTimes self.wait_for_exposure_callable = wait_for_exposure_callable self.configureLock = threading.Lock() self.dataset = None
def __init__(self, args): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) self.wh_con = zxJDBC.connect(args[Constant.WH_DB_URL_KEY], args[Constant.WH_DB_USERNAME_KEY], args[Constant.WH_DB_PASSWORD_KEY], args[Constant.WH_DB_DRIVER_KEY]) self.wh_cursor = self.wh_con.cursor() self.app_id = int(args[Constant.JOB_REF_ID_KEY]) self.group_app_id = int(args[Constant.LDAP_GROUP_APP_ID_KEY]) self.app_folder = args[Constant.WH_APP_FOLDER_KEY] self.metadata_folder = self.app_folder + "/" + str(self.app_id) self.ceo_user_id = args[Constant.LDAP_CEO_USER_ID_KEY]
def __init__(self, args): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) self.app_id = int(args[Constant.JOB_REF_ID_KEY]) self.wh_exec_id = long(args[Constant.WH_EXEC_ID_KEY]) self.wh_con = zxJDBC.connect(args[Constant.WH_DB_URL_KEY], args[Constant.WH_DB_USERNAME_KEY], args[Constant.WH_DB_PASSWORD_KEY], args[Constant.WH_DB_DRIVER_KEY]) self.wh_cursor = self.wh_con.cursor() self.look_back_days = args[Constant.AW_LINEAGE_ETL_LOOKBACK_KEY] self.last_execution_unix_time = None self.get_last_execution_unix_time()
def __init__(self): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) requests.packages.urllib3.disable_warnings() self.app_id = int(args[Constant.APP_ID_KEY]) self.wh_exec_id = long(args[Constant.WH_EXEC_ID_KEY]) self.project_writer = FileWriter(args[Constant.GIT_PROJECT_OUTPUT_KEY]) self.repo_writer = FileWriter(args[Constant.PRODUCT_REPO_OUTPUT_KEY]) self.repo_owner_writer = FileWriter(args[Constant.PRODUCT_REPO_OWNER_OUTPUT_KEY]) self.multiproduct = {} self.git_repo = {} self.product_repo = []
def __init__(self): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) # connection username = args[Constant.HIVE_METASTORE_USERNAME] password = args[Constant.HIVE_METASTORE_PASSWORD] jdbc_driver = args[Constant.HIVE_METASTORE_JDBC_DRIVER] jdbc_url = args[Constant.HIVE_METASTORE_JDBC_URL] self.conn_hms = zxJDBC.connect(jdbc_url, username, password, jdbc_driver) self.curs = self.conn_hms.cursor() # variable self.dataset_dict = {}
def getLogLevel(loggerName='ROOT'): logger = LoggerFactory.getLogger(loggerName) if logger.getLevel(): level = str(logger.getLevel()) else: level = None if logger.getEffectiveLevel(): effectiveLevel = str(logger.getEffectiveLevel()) else: effectiveLevel = None return { 'level': level, 'effectiveLevel': effectiveLevel }
def __init__(self, args): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) # connection self.username = args[Constant.HIVE_METASTORE_USERNAME] self.password = args[Constant.HIVE_METASTORE_PASSWORD] self.jdbc_driver = args[Constant.HIVE_METASTORE_JDBC_DRIVER] self.jdbc_url = args[Constant.HIVE_METASTORE_JDBC_URL] self.connection_interval = int(args[Constant.HIVE_METASTORE_RECONNECT_TIME]) self.logger.info("DB re-connection interval: %d" % self.connection_interval) self.db_whitelist = args[Constant.HIVE_DATABASE_WHITELIST_KEY] if Constant.HIVE_DATABASE_WHITELIST_KEY in args else "''" self.db_blacklist = args[Constant.HIVE_DATABASE_BLACKLIST_KEY] if Constant.HIVE_DATABASE_BLACKLIST_KEY in args else "''" self.logger.info("DB whitelist: " + self.db_whitelist) self.logger.info("DB blacklist: " + self.db_blacklist) self.conn_hms = None self.connect_time = None self.db_connect(True) hdfs_namenode_ipc_uri = args.get(Constant.HDFS_NAMENODE_IPC_URI_KEY, None) kerberos_principal = args.get(Constant.KERBEROS_PRINCIPAL_KEY, None) keytab_file = args.get(Constant.KERBEROS_KEYTAB_FILE_KEY, None) kerberos_auth = False if Constant.KERBEROS_AUTH_KEY in args: kerberos_auth = FileUtil.parse_bool(args[Constant.KERBEROS_AUTH_KEY], False) self.table_whitelist_enabled = False if Constant.HIVE_TABLE_WHITELIST_ENABLED in args: self.table_whitelist_enabled = FileUtil.parse_bool(args[Constant.HIVE_TABLE_WHITELIST_ENABLED], False) self.table_blacklist_enabled = False if Constant.HIVE_TABLE_BLACKLIST_ENABLED in args: self.table_blacklist_enabled = FileUtil.parse_bool(args[Constant.HIVE_TABLE_BLACKLIST_ENABLED], False) self.schema_url_helper = SchemaUrlHelper.SchemaUrlHelper(hdfs_namenode_ipc_uri, kerberos_auth, kerberos_principal, keytab_file) # global variables self.databases = None self.db_dict = {} # name : index self.table_dict = {} # fullname : index self.dataset_dict = {} # name : index self.instance_dict = {} # name : index self.serde_param_columns = [] # counting statistics self.external_url = 0 self.hdfs_count = 0 self.schema_registry_count = 0
def __init__(self, args): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) self.args = args self.app_id = int(args[Constant.APP_ID_KEY]) self.group_app_id = int(args[Constant.LDAP_GROUP_APP_ID_KEY]) self.wh_exec_id = long(args[Constant.WH_EXEC_ID_KEY]) self.app_folder = args[Constant.WH_APP_FOLDER_KEY] self.metadata_folder = self.app_folder + "/" + str(self.app_id) if not os.path.exists(self.metadata_folder): try: os.makedirs(self.metadata_folder) except Exception as e: self.logger.error(e) self.ldap_user = set() self.group_map = dict() self.group_flatten_map = dict()
def __init__(self, name, rootNamespace=None, scannablesToRead=[], readFromNexus=False): """Create a MetadataCollector Scannable, for use with SRSDataWriters """ self.name = name self.inputNames = [] self.extraNames = [] self.outputFormat = [] self.logger = LoggerFactory.getLogger("metadata") if scannablesToRead and readFromNexus: self.logger.warn("%s: When readFromNexus=True the specified scannablesToRead are ignored!" % name) self.readFromNexus = readFromNexus self.rootNamespaceDict = rootNamespace self.scannables_to_read = scannablesToRead self.verbose = False self.quiet = False self.prepend_keys_with_scannable_names = True
def __init__(self, hdfs_uri, kerberos=False, kerberos_principal=None, keytab_file=None): """ :param hdfs_uri: hdfs://hadoop-name-node:port :param kerberos: optional, if kerberos authentication is needed :param kerberos_principal: optional, [email protected] :param keytab_file: optional, user.keytab or ~/.kerberos/user.keytab """ self.logger = LoggerFactory.getLogger(self.__class__.__name__) hdfs_conf = Configuration() if hdfs_uri.startswith('hdfs://'): hdfs_conf.set(Hdfs.FS_DEFAULT_NAME_KEY, hdfs_uri) elif hdfs_uri > "": self.logger.error("%s is an invalid uri for hdfs namenode ipc bind." % hdfs_uri) if kerberos == True: # init kerberos and keytab if not kerberos_principal or not keytab_file or kerberos_principal == '' or keytab_file == '': print "Kerberos Principal and Keytab File Name/Path are required!" keytab_path = keytab_file if keytab_file.startswith('/'): if os.path.exists(keytab_file): keytab_path = keytab_file print "Using keytab at %s" % keytab_path else: # try relative path all_locations = [os.getcwd(), expanduser("~") + "/.ssh", expanduser("~") + "/.kerberos", expanduser("~") + "/.wherehows", os.getenv("APP_HOME"), os.getenv("WH_HOME")] for loc in all_locations: if os.path.exists(loc + '/' + keytab_file): keytab_path = loc + '/' + keytab_file print "Using keytab at %s" % keytab_path break hdfs_conf.set("hadoop.security.authentication", "kerberos") hdfs_conf.set("dfs.namenode.kerberos.principal.pattern", "*") UserGroupInformation.setConfiguration(hdfs_conf) UserGroupInformation.loginUserFromKeytab(kerberos_principal, keytab_path) self.fs = Hdfs.get(hdfs_conf) requests.packages.urllib3.disable_warnings()
def __init__(self, file_content=None, log_file_name=None): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) self.text = None if log_file_name is not None: self.log_file_name = log_file_name file_ext = os.path.splitext(self.log_file_name)[1][1:] self.logger.info(file_ext) self.logger.info(self.log_file_name) if file_ext == 'gz': try: self.text = gzip.GzipFile(mode='r', filename=self.log_file_name).read() except: self.logger.error('exception') self.logger.error(str(sys.exc_info()[0])) else: self.text = open(self.log_file_name,'r').read() else: self.text = file_content
def __init__(self, args): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) self.app_id = int(args[Constant.JOB_REF_ID_KEY]) self.wh_exec_id = long(args[Constant.WH_EXEC_ID_KEY]) self.aw_con = zxJDBC.connect(args[Constant.WH_DB_URL_KEY], args[Constant.WH_DB_USERNAME_KEY], args[Constant.WH_DB_PASSWORD_KEY], args[Constant.WH_DB_DRIVER_KEY]) self.aw_cursor = self.aw_con.cursor() self.remote_hadoop_script_dir = args[Constant.AW_REMOTE_HADOOP_SCRIPT_DIR] self.local_script_path = args[Constant.AW_LOCAL_SCRIPT_PATH] self.remote_script_path = args[Constant.AW_REMOTE_SCRIPT_PATH] self.aw_archive_dir = args[Constant.AW_ARCHIVE_DIR] # self.aw_log_url = args[Constant.AW_LOG_URL] self.bteq_source_target_override = args[Constant.AW_BTEQ_SOURCE_TARGET_OVERRIDE] self.metric_override = args[Constant.AW_METRIC_OVERRIDE] self.skip_already_parsed = args[Constant.AW_SKIP_ALREADY_PARSED] self.look_back_days = args[Constant.AW_LINEAGE_ETL_LOOKBACK_KEY] self.last_execution_unix_time = None self.get_last_execution_unix_time()
def __init__(self, args): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) username = args[Constant.WH_DB_USERNAME_KEY] password = args[Constant.WH_DB_PASSWORD_KEY] JDBC_DRIVER = args[Constant.WH_DB_DRIVER_KEY] JDBC_URL = args[Constant.WH_DB_URL_KEY] self.database_scm_repo_file = args[Constant.DATABASE_SCM_REPO_OUTPUT_KEY] self.app_id = args[Constant.APP_ID_KEY] self.wh_etl_exec_id = args[Constant.WH_EXEC_ID_KEY] self.conn_mysql = zxJDBC.connect(JDBC_URL, username, password, JDBC_DRIVER) self.conn_cursor = self.conn_mysql.cursor() if Constant.INNODB_LOCK_WAIT_TIMEOUT in args: lock_wait_time = args[Constant.INNODB_LOCK_WAIT_TIMEOUT] self.conn_cursor.execute("SET innodb_lock_wait_timeout = %s;" % lock_wait_time) self.logger.info("Load Code Search CSV into {}, app_id {}, wh_exec_id {}" .format(JDBC_URL, self.app_id, self.wh_etl_exec_id))
def __init__(self, args): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) self.app_id = int(args[Constant.JOB_REF_ID_KEY]) self.wh_exec_id = long(args[Constant.WH_EXEC_ID_KEY]) self.oz_con = zxJDBC.connect(args[Constant.OZ_DB_URL_KEY], args[Constant.OZ_DB_USERNAME_KEY], args[Constant.OZ_DB_PASSWORD_KEY], args[Constant.OZ_DB_DRIVER_KEY]) self.oz_cursor = self.oz_con.cursor() self.lookback_period = args[Constant.OZ_EXEC_ETL_LOOKBACK_MINS_KEY] self.app_folder = args[Constant.WH_APP_FOLDER_KEY] self.metadata_folder = self.app_folder + "/" + str(SchedulerType.OOZIE) + "/" + str(self.app_id) self.oz_version = 4.0 if not os.path.exists(self.metadata_folder): try: os.makedirs(self.metadata_folder) except Exception as e: self.logger.error(e) self.get_oozie_version()
def getLogLevel(loggerName='ROOT'): """Return log level. Optional Args: loggerName: Name of a specific 'logger'. Default value is 'ROOT'. Returns: Dictionary containing both the 'level' and 'effectiveLevel' of the given logger. """ logger = LoggerFactory.getLogger(loggerName) if logger.getLevel(): level = str(logger.getLevel()) else: level = None if logger.getEffectiveLevel(): effectiveLevel = str(logger.getEffectiveLevel()) else: effectiveLevel = None return { 'level': level, 'effectiveLevel': effectiveLevel }
def __init__(self, args): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) username = args[Constant.WH_DB_USERNAME_KEY] password = args[Constant.WH_DB_PASSWORD_KEY] JDBC_DRIVER = args[Constant.WH_DB_DRIVER_KEY] JDBC_URL = args[Constant.WH_DB_URL_KEY] self.mp_gitli_project_file = args[Constant.GIT_PROJECT_OUTPUT_KEY] self.product_repo_file = args[Constant.PRODUCT_REPO_OUTPUT_KEY] self.product_repo_owner_file = args[Constant.PRODUCT_REPO_OWNER_OUTPUT_KEY] self.app_id = args[Constant.APP_ID_KEY] self.wh_etl_exec_id = args[Constant.WH_EXEC_ID_KEY] self.conn_mysql = zxJDBC.connect(JDBC_URL, username, password, JDBC_DRIVER) self.conn_cursor = self.conn_mysql.cursor() if Constant.INNODB_LOCK_WAIT_TIMEOUT in args: lock_wait_time = args[Constant.INNODB_LOCK_WAIT_TIMEOUT] self.conn_cursor.execute("SET innodb_lock_wait_timeout = %s;" % lock_wait_time) self.logger.info("Load Multiproduct Metadata into {}, app_id {}, wh_exec_id {}" .format(JDBC_URL, self.app_id, self.wh_etl_exec_id))
def tryToLoadDataset(path, iFileLoader): # Return None if the dataset could not be loaded for any reason logger = LoggerFactory.getLogger("DatasetProvider.tryToLoadDataset") if not os.path.exists(path): logger.info('Path {} does not exist, returning None', path) return None try: dataset = loadImageIntoSFH(path, iFileLoader)[0] # a dataset if not len(dataset.shape) == 2: logger.error('Expected 2 dimensions but found %r when sanity checking image %r using loader %r' % (dataset.shape, path, iFileLoader)) print "*" * 80 print "DatasetProvider.tryToLoadDataset got a dataset with ", dataset.shape, " dimensions." print "The analysis code will try again to load the image, and unless it times out everything is *OKAY*" print "Please call DASC support to report this (tricky to track down) bug" print "*" * 80 return None logger.debug('Returning dataset {} from path {}', dataset, path) return dataset except: logger.error('Error loading or sanity checking image %r using loader %r :\n %s' % (path, iFileLoader, ''.join(traceback.format_exception(*sys.exc_info())))) return None
def __init__(self, wh_etl_exec_id='0'): self.logger = LoggerFactory.getLogger("%s[%s]" % (self.__class__.__name__, wh_etl_exec_id)) # set up connection username = args[Constant.WH_DB_USERNAME_KEY] password = args[Constant.WH_DB_PASSWORD_KEY] JDBC_DRIVER = args[Constant.WH_DB_DRIVER_KEY] JDBC_URL = args[Constant.WH_DB_URL_KEY] self.conn_mysql = zxJDBC.connect(JDBC_URL, username, password, JDBC_DRIVER) self.conn_cursor = self.conn_mysql.cursor() if Constant.INNODB_LOCK_WAIT_TIMEOUT in args: lock_wait_time = args[Constant.INNODB_LOCK_WAIT_TIMEOUT] self.conn_cursor.execute("SET innodb_lock_wait_timeout = %s;" % lock_wait_time) temp_dir = FileUtil.etl_temp_dir(args, "HIVE") self.input_schema_file = os.path.join(temp_dir, args[Constant.HIVE_SCHEMA_CSV_FILE_KEY]) self.input_field_file = os.path.join(temp_dir, args[Constant.HIVE_FIELD_METADATA_KEY]) self.input_instance_file = os.path.join(temp_dir, args[Constant.HIVE_INSTANCE_CSV_FILE_KEY]) self.input_dependency_file = os.path.join(temp_dir, args[Constant.HIVE_DEPENDENCY_CSV_FILE_KEY]) self.db_id = args[Constant.JOB_REF_ID_KEY] self.wh_etl_exec_id = args[Constant.WH_EXEC_ID_KEY]
def __init__(self, args): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) self.elasticsearch_server_url = args[Constant.ELASTICSEARCH_URL_KEY] self.elasticsearch_port = args[Constant.ELASTICSEARCH_PORT_KEY] if Constant.ELASTICSEARCH_INDEX_KEY not in args: self.elasticsearch_index = "wherehows" else: self.elasticsearch_index = args[Constant.ELASTICSEARCH_INDEX_KEY] self.index_mapping_file = args[Constant.WH_ELASTICSEARCH_INDEX_MAPPING_FILE] self.bulk_chunk_size = int(args[Constant.ELASTICSEARCH_BULK_INSERT_SIZE]) # bulk insert size to elastic search engine self.es_url_request_timeout = int(args[Constant.ELASTICSEARCH_URL_REQUEST_TIMEOUT]) # url to post data to elastic search engine request time out self.max_retry_times = int(args[Constant.WH_DB_MAX_RETRY_TIMES]) # max times for db re-connection when lost during fetching source data self.base_url = self.elasticsearch_server_url + ':' + str(self.elasticsearch_port) + '/' self.logger.info(self.base_url) self.old_index = [] self.new_index = [] self.databaseConnect(args)
from org.slf4j import LoggerFactory LOG = LoggerFactory.getLogger("jython.Startup") from java.lang import System import sys, platform LOG.warn("") if hasattr(sys.version_info, "major"): LOG.warn("Jython version: {}.{}.{}.{}".format(sys.version_info.major, sys.version_info.minor, sys.version_info.micro, sys.version_info.releaselevel)) else: LOG.warn("Jython version: {}".format(sys.version)) LOG.warn("Operating system: {}".format(System.getProperty("os.name"))) LOG.warn("OS Version: {}".format(System.getProperty("os.version"))) LOG.warn("Architecture: {}".format(platform.uname()[5])) LOG.warn("Java version: {}".format(sys.platform)) LOG.warn("sys.path: {}".format(sys.path)) LOG.warn("")
''' Process presence contact change event. A presence event sets uses the Open event to signal occupancy The Closed event ends the occupancy. Area is considered occupied until the event ends. No other items can change the occupancy state of the area. Convert Open/Closed events to begin/end events for the area item ''' from org.slf4j import Logger, LoggerFactory log = LoggerFactory.getLogger("org.eclipse.smarthome.model.script.Rules") import personal.occupancy.areas.events.event_base reload(personal.occupancy.areas.events.event_base) from personal.occupancy.areas.events.event_base import Event_Base class Event_Contact_Presence(Event_Base): def process_changed_event(self, event): Event_Base.process_changed_event(self, event) item_state = str(event.itemState) if item_state == "OPEN": #begin event self.begin_event(event) elif item_state == "CLOSED": # off, an end event, only change occupancy settings if specified self.end_event(event)
def __init__(self): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__) self.base_url = args[Constant.BASE_URL_KEY] self.code_search_committer_writer = FileWriter( args[Constant.DATABASE_SCM_REPO_OUTPUT_KEY])
def __init__( self, name, pgm_grat_pitch, pgm_mirr_pitch, pgmpvroot, id_energy, idpvroot, move_pgm=True, move_id=True): # motors, maybe also detector to set the delay time self.logger = LoggerFactory.getLogger( "ContinuousPgmGratingIDGapEnergyMoveController:%s" % name) self.verbose = False self.setName(name) self._start_event = threading.Event() self._movelog_time = datetime.now() #PGM self._pgm_grat_pitch = pgm_grat_pitch self._pgm_mirr_pitch = pgm_mirr_pitch self._pgm_grat_pitch_speed_orig = None self._pgm_runupdown_time = None self._move_pgm = move_pgm self._move_id = move_id self.pvs = PvManager( { 'grating_density': 'NLINES', 'cff': 'CFF', 'grating_offset': 'GRTOFFSET', 'plane_mirror_offset': 'MIROFFSET', 'pgm_energy': 'ENERGY', 'grating_pitch': 'GRT:PITCH', 'mirror_pitch': 'MIR:PITCH', 'energy_calibration_gradient': 'MX', 'energy_calibration_reference': 'REFERENCE' }, pgmpvroot) if installation.isLive(): self.pvs.configure() #ID self._id_energy = id_energy self._id_gap = self._id_energy.id_gap self._id_gap_speed_orig = None self._id_runupdown_time = None self.idpvs = PvManager({ 'vel': 'BLGSETVEL', 'acc': 'IDGSETACC' }, idpvroot) if installation.isLive(): self.idpvs.configure() self.grating_pitch_positions = [] self.mirror_pitch_positions = [] self.pgm_energy_positions = [] self._start_time = None self.idspeedfactor = 1.0 self.pgmspeedfactor = 1.0 self.idstartdelaytime = 0.0 self._move_start = 0.0 self._move_end = 1.0 self._move_step = 0.1 self._triggerPeriod = 0.0 self._pgm_runupdown_time = 0.0
def __init__(this): this.logger = LoggerFactory.getLogger('jython script : ' + this.__class__.__name__)
def __init__(self): self.logger = LoggerFactory.getLogger("meetup.Client") self.logger.error("meetup.Client Created ==================") return
# # THIS CODE AND INFORMATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS # FOR A PARTICULAR PURPOSE. THIS CODE AND INFORMATION ARE NOT SUPPORTED BY XEBIALABS. # import mimetypes import random import string import sys import json from xlrelease.HttpRequest import HttpRequest import org.slf4j.LoggerFactory as LoggerFactory logger = LoggerFactory.getLogger("Blazemeter") _BOUNDARY_CHARS = string.digits + string.ascii_letters # Encode multipart form data to upload files via POST def encode_multipart(fields, files, boundary=None): r"""Encode dict of form fields and dict of files as multipart/form-data. Return tuple of (body_string, headers_dict). Each value in files is a dict with required keys 'filename' and 'content', and optional 'mimetype' (if not specified, tries to guess mime type or uses 'application/octet-stream'). >>> body, headers = encode_multipart({'FIELD': 'VALUE'}, ... {'FILE': {'filename': 'F.TXT', 'content': 'CONTENT'}}, ... boundary='BOUNDARY') >>> print('\n'.join(repr(l) for l in body.split('\r\n'))) '--BOUNDARY' 'Content-Disposition: form-data; name="FIELD"' ''
def __init__(self, server, username, password): self.logger = LoggerFactory.getLogger("com.xebialabs.bitbucket-plugin") creds = CredentialsFallback(server, username, password).getCredentials() self.http_request = HttpRequest(server, creds['username'], creds['password'])
def __init__(self, scannable_to_read): self.logger = LoggerFactory.getLogger("PositionInputStreamImplementer") self.scannable=scannable_to_read self.verbose=True
def setLogLevel(loggerName="console", logLevel="DEBUG"): loggerMap = {} logLevel = logLevel.upper() loggerContext = LoggerFactory.getILoggerFactory() loggerList = loggerContext.getLoggerList() for loggerItem in loggerList: if (loggerItem.getName() == loggerName): myLogger.info("Setting %s to %s" % (loggerName, logLevel)) loggerItem.setLevel(logLevels.toLevel(logLevel)) myLogger.info("%s = %s" % (loggerName, logLevel)) #myLogger.error("%s != %s" % (loggerItem.getName(), loggerName)) return myLogger = LoggerFactory.getLogger("logmanager") verb = "GET" if (request): if (request.query): if (request.query['verb']): verb = request.query['verb'] if (verb == "SET"): loggerName = request.query['logger'] logLevel = request.query['level'] myLogger.info("Setting %s to %s" % (loggerName, logLevel)) setLogLevel(loggerName, logLevel) loggerMap = getLogLevel() #loggerMap = {}
import requests from requests.auth import HTTPBasicAuth import json import time import datetime #import com.xebialabs.xlrelease.api.v1 #import com.xebialabs.xlrelease.domain from java.time import LocalDate, ZoneId # beter way to do this import java.util.Date import dateutil.parser import org.slf4j.LoggerFactory as LoggerFactory # import com.xebialabs.xlrelease.api.v1 as releaseApi import com.xebialabs.xlrelease.api.XLReleaseServiceHolder as XLReleaseServiceHolder logger = LoggerFactory.getLogger("Planner") releaseApi = XLReleaseServiceHolder.getReleaseApi() phaseApi = XLReleaseServiceHolder.getPhaseApi() taskApi = XLReleaseServiceHolder.getTaskApi() MANUAL_TASK_DURATION = 60 * 60 * 1000 AUTOMATED_TASK_DURATION = 1 * 60 * 1000 hardcodedVars = [ "Release-Ready", "CodeFrozen-Flag", "PendSecurityScan", "Dependency-Flag", "NoOpenDefects-Flag", "BXImpact-Flag" ] class Planner(object): def __init__(self):
def getLogger(cls): if grinder: logger = grinder.logger return logger return LoggerFactory.getLogger('root')
def __init__(self): # motors, maybe also detector to set the delay time self.logger = LoggerFactory.getLogger("TrajectoryControllerHelper")
# # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # import sys import time import com.xhaus.jyson.JysonCodec as json from xlrelease.HttpRequest import HttpRequest from quickbuild.QuickBuild import QuickBuild from org.slf4j import Logger from org.slf4j import LoggerFactory logger = LoggerFactory.getLogger("quickbuild") logger.error("===============================================================") qb = QuickBuild.create_client(qbServer) logger.warn(">> configurationId = %s " % configurationId) logger.warn(">> respectBuildCondition = %s " % respectBuildCondition) for prop in variables.keys(): logger.warn(">> %s = %s " % (prop, variables[prop])) #qb = QuickBuild.create_client( params ) logger.error("===============================================================") if len(variables) > 0: buildVariables = "\n<variables>" for prop in variables.keys(): buildVariables = "%s\n<entry>\n<string>%s</string>\n<string>%s</string>\n</entry>" % (
def __init__(self): self.logger = LoggerFactory.getLogger('jython script : ' + self.__class__.__name__)
# Origin of script: https://community.openhab.org/t/solved-jython-jsr223-not-properly-initialised-during-startup/46031/6 # Issue: https://github.com/eclipse/smarthome/issues/4324 # This script works around an init issue above that causes this error: # TypeError: can't set attributes of built-in/extension type 'NoneType' in <script> at line number 3 import time from org.slf4j import Logger, LoggerFactory log = LoggerFactory.getLogger("org.eclipse.smarthome.automation") log.info("jsr223: checking for initialised context") time.sleep(30) while True: try: scriptExtension.importPreset("RuleSupport") if automationManager != None: break except: pass log.info( "jsr223: context not initialised yet. waiting 10 sec before checking again" ) time.sleep(40) log.info("jsr223: done")
#from requests.auth import HTTPBasicAuth import json import sets import datetime import time import dateutil.parser import dateutil.parser as dp from java.time import LocalDate, ZoneId import com.xebialabs.xlrelease.api.XLReleaseServiceHolder as XLReleaseServiceHolder import com.xebialabs.xlrelease.api.v1.forms.ReleasesFilters as ReleasesFilters # import com.xebialabs.xlrelease.api.v1.FolderApi as folderApi # import com.xebialabs.xlrelease.api.v1.FolderApi as folderApi import org.slf4j.LoggerFactory as LoggerFactory logger = LoggerFactory.getLogger("David Tile") import planner.planner reload(planner.planner) from planner.planner import Planner the_planner = Planner() HTTP_SUCCESS = sets.Set([200, 201, 202, 203, 204, 205, 206, 207, 208]) HTTP_ERROR = sets.Set([ 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 412, 413, 414, 415 ]) filterReleaseTags = list(releaseTags) filterReleaseFolders = list(folderFilter) filterStatuses = ["FAILED", "COMPLETED", "SKIPPED", "ABORTED"]
def __init__(self, server, username, password): self.logger = LoggerFactory.getLogger("com.xebialabs.bitbucket-plugin") if username not in [None, ""] or password not in [None, ""]: self.http_request = HttpRequest(server, username, password) else: self.http_request = HttpRequest(server)
# # Copyright 2020 XEBIALABS # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import json import org.slf4j.LoggerFactory as LoggerFactory logger = LoggerFactory.getLogger("Counter") InfrastructureCount = 0 DeploymentCount = 0 infraSum = [] appSum = [] def get_deployables(): master_list = [] applications = repositoryService.query(None, "Applications", "", "", None, None, 0, 0) deployment_list = [] deployables = [] applications = get_directories(applications) master_list = convert(applications) for x in master_list: deployment_list = (repositoryService.query(None, x['id'], "", "", None, None, 0, 0)) deployment_list = remove_CompositePackages(deployment_list) x['deployment_list'] = convert(deployment_list)
def __init__(self): self.logger = LoggerFactory.getLogger("ScriptLogger")
from gda.device.detector import DetectorBase from gda.configuration.properties import LocalProperties import gda.data as data from gda.jython import InterfaceProvider import rpiComms import time from org.slf4j import LoggerFactory logger = LoggerFactory.getLogger(__name__ + '.py') class rpiCameraScannable(DetectorBase): ## scan info - ScanInformation scanInfo = InterfaceProvider.getCurrentScanInformationHolder().getCurrentScanInformation(); def __init__(self, name): logger.debug("Camera Setup") self.pin = -1 self.device = name self.setName(name) self.currentPosition = 0 self.lastPosition = 0 # required self.busyStatus = False beamlineName = LocalProperties.get("gda.beamline.name") self.numTracker = data.NumTracker(beamlineName) rpiComms.rpiCommunicator.scannables.append(self) def collectData(self): self.busyStatus = True self.lastPosition = self.currentPosition rpiComms.commController.outgoingQueue.put("-1,c" + self.device + ",CAPTURE,None,0//")
# Simulator for Dispatcher's AutoActiveTrains # while auto train(s) are "moving", repeatedly activate "next" allocated block, and deactivate "last" occupied block # waits for debounce time plus a bit, to allow signals, etc. to respond. # Runs as a background thread, ends itself when no trains are found in Dispatcher Active Trains list. # NOTE: to enable logging, add "log4j.category.jmri.jmrit.jython.exec=DEBUG" to default.lcf import jmri import time from org.slf4j import Logger from org.slf4j import LoggerFactory log = LoggerFactory.getLogger( "jmri.jmrit.jython.exec.AutoActiveTrains_Simulator") # create a new class to run as thread class AutoActiveTrains_Simulator(jmri.jmrit.automat.AbstractAutomaton): # def init(self): def handle(self): DF = jmri.jmrit.dispatcher.DispatcherFrame.instance() trainsList = DF.getActiveTrainsList() #loop thru all trains if ( trainsList.size() == 0 ): # kill the thread if no trains found TODO: add something outside to restart log.info("AutoActiveTrains_Simulator thread ended") return False # no trains, end totDelay = 0 # keep track of delay time to give CPU some time for other stuff for i in range(trainsList.size()): at = trainsList.get(i) #: :type at: ActiveTrain
# Add some static files to serve. These take priority over regex matches. handler.static("/base.source", "templates/base.template", contentType="text/plain; charset=utf-8") handler.static("/base.source.cache", "templates/base.template", contentType="text/plain; charset=utf-8", cache=True) # Create a logger, (kind of hacky code) class PythonCode(JavaObject): pass log = LoggerFactory.getLogger(PythonCode) @handler.path("/name/(?P<name>\\w+)") @template("hello.template") def handle_name_page(pyRequest): name = pyRequest.getStrGroup("name", "Admin") pyRequest.out.println(pyRequest.template.render({"name": name})) @handler.path("/socketio") @template("socketio.template") def handle_socketIO_front(pyRequest): plugins = __bird__.getActiveMockingBirdREPL().getPlugins()
def __init__(self, wh_etl_exec_id='0'): self.logger = LoggerFactory.getLogger("%s[%s]" % (self.__class__.__name__, wh_etl_exec_id))
# # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # from bitbucket.Bitbucket import BitbucketClient import json import time from java.time import LocalDate, ZonedDateTime import org.slf4j.Logger as Logger import org.slf4j.LoggerFactory as LoggerFactory logger = LoggerFactory.getLogger("com.xebialabs.bitbucket-plugin") def convertRFC3339ToDate(timestamp): zonedDateTime = ZonedDateTime.parse(timestamp) return zonedDateTime.toLocalDate() bitbucket = BitbucketClient.get_client(server, username, password) commits = bitbucket.bitbucket_querycommits(locals()) # Compile data for summary view commitsByDay = {} for commit in commits: logger.warn("commit date %s" % commit["date"]) commitDate = convertRFC3339ToDate(commit["date"])
def __init__(self, exec_id): self.logger = LoggerFactory.getLogger(self.__class__.__name__ + ':' + str(exec_id))
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.kafka.core.KafkaTemplate; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @RestController @RequestMapping("/kafka") public class CollectController { protected final Logger logger = LoggerFactory.getLogger(this.getClass()); @Autowired private KafkaTemplate kafkaTemplate; @RequestMapping(value = "/send") public String sendKafka(HttpServletRequest request, HttpServletResponse response,String message) { try { logger.info("kafka的消息={}", message); kafkaTemplate.send("test", "key", message);//test作为topic,key配置为key,message再页面进行发送,进行测试 logger.info("发送kafka成功."); return "发送kafka成功"; } catch (Exception e) { logger.error("发送kafka失败", e); return "发送kafka失败"; } }
class OmnikOpenhab(object): """ Get data from Omniksol inverter and store the data in OpenHAB items. """ global logger logger = LoggerFactory.getLogger( "org.eclipse.smarthome.model.script.rules") def __init__(self, config_file): #logger.info("omnikOPENHAB - Starting") # Load the settings path = "/etc/openhab2/automation/jsr223/" config_files = path + config_file self.config = ConfigParser.RawConfigParser() self.config.read(config_files) def getInverters(self): #Get number of inverters inverterCount = len(self.config.sections()) - 2 #logger.info("omnikOPENHAB - Invertercount: {0}".format(inverterCount)) #Reset totals to zero OmnikOpenhab.total_e_today = 0 OmnikOpenhab.total_e_total = 0 OmnikOpenhab.total_p_ac = 0 #For each inverter, get data and add to total for i in range(1, inverterCount + 1): #logger.info("omnikOPENHAB - In the inverterloop, value of index: {0}".format(i)) msg = self.run(i) #logger.info("omnikOPENHAB - Value of msg is {0}".format(msg)) #Assume daytime (for processing data) day = 1 # If retrieved data is not a InverterMsg object: No updates #if 'timed' in format(msg): if isinstance(msg, InverterMsg): #logger.info("omnikOPENHAB - Day time -- valid data") self.add(msg) else: day = 0 #logger.info("omnikOPENHAB - Timed out - BREAKING") break #Only process durig day time if day: etotal = self.config.get('openhab_items', 'etotal') etoday = self.config.get('openhab_items', 'etoday') epower = self.config.get('openhab_items', 'epower') logger.info( "omnikOPENHAB - Items updated: {0}: {1}, {2}: {3}, {4}: {5}, ". format(etotal, OmnikOpenhab.total_e_total, etoday, OmnikOpenhab.total_e_today, epower, OmnikOpenhab.total_p_ac)) events.postUpdate(str(etotal), str(OmnikOpenhab.total_e_total)) events.postUpdate(str(etoday), str(OmnikOpenhab.total_e_today)) events.postUpdate(str(epower), str(OmnikOpenhab.total_p_ac)) else: logger.info( "omnikOPENHAB - No data (after sunset?), not updating database" ) #logger.info("omnikOPENHAB - End") def add(self, msg): #logger.info("omnikOPENHAB - Adding data") OmnikOpenhab.total_e_today += msg.e_today OmnikOpenhab.total_e_total += msg.e_total OmnikOpenhab.total_p_ac += msg.p_ac(1) + msg.p_ac(2) + msg.p_ac(3) def run(self, inverternr): """Get information from inverter and store is configured outputs.""" # Connect to inverter msg = '' ip = self.config.get('inverter' + str(inverternr), 'ip') port = self.config.get('inverter' + str(inverternr), 'port') for res in socket.getaddrinfo(ip, port, socket.AF_INET, socket.SOCK_STREAM): family, socktype, proto, canonname, sockadress = res try: #logger.info("omnikOPENHAB - connecting to {0} port {1}".format(ip, port)) inverter_socket = socket.socket(family, socktype, proto) inverter_socket.settimeout(10) inverter_socket.connect(sockadress) #logger.info("omnikOPENHAB - Retrieved data..") except socket.error as msg: return (msg) #logger.info("omnikOPENHAB - Could not connect to inverter.") wifi_serial = self.config.getint('inverter' + str(inverternr), 'wifi_sn') inverter_socket.sendall(OmnikOpenhab.generate_string(wifi_serial)) data = inverter_socket.recv(1024) inverter_socket.close() msg = InverterMsg(data) return (msg) def override_config(self, section, option, value): """Override config settings""" self.config.set(section, option, value) @staticmethod def generate_string(serial_no): """Create request string for inverter. The request string is build from several parts. The first part is a fixed 4 char string; the second part is the reversed hex notation of the s/n twice; then again a fixed string of two chars; a checksum of the double s/n with an offset; and finally a fixed ending char. Args: serial_no (int): Serial number of the inverter Returns: str: Information request string for inverter """ response = '\x68\x02\x40\x30' double_hex = hex(serial_no)[2:] * 2 hex_list = [ double_hex[i:i + 2].decode('hex') for i in reversed(range(0, len(double_hex), 2)) ] cs_count = 115 + sum([ord(c) for c in hex_list]) checksum = hex(cs_count)[-2:].decode('hex') response += ''.join(hex_list) + '\x01\x00' + checksum + '\x16' return response
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # ''' Calls Snyk API to query for projects owned by OrgId ''' v = vars() import org.slf4j.Logger as Logger import org.slf4j.LoggerFactory as LoggerFactory import httplib import ssl from com.xhaus.jyson import JysonCodec as json logger = LoggerFactory.getLogger('snyk-plugin') logger.debug("Starting GetProjects.py") base_url = str(valueProvider.server.url) token = str(valueProvider.server.token) orgId = str(valueProvider.server.orgId) snyk_host = str(base_url.split("/")[2]) path = '/' + str(base_url.split("/")[3]) + '/' + str( base_url.split("/")[4]) + '/org/' + orgId + '/projects' headers = { "Content-Type": "application/json", "Authorization": "token {}".format(token) } uv_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) connection = httplib.HTTPSConnection(snyk_host, context=uv_context)
# # Copyright 2020 XEBIALABS # # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # import stash.Stash if classReload: reload(stash.Stash) from stash.Stash import StashClient import org.slf4j.LoggerFactory as LoggerFactory import org.slf4j.Logger as Logger logger = LoggerFactory.getLogger("Stash") stash = StashClient.get_client(server, username, password) method = str(task.getTaskType()).lower().replace('.', '_') logger.error("Call Stash Method %s" % method) call = getattr(stash, method) response = call(locals()) for key, value in response.items(): locals()[key] = value