class myRepo(object): def __init__(self, version_log="", root_path=""): self.root_dir = os.getcwd() self.release_dist_dir = '' self.build_root_dir = '' self.git_root_dir = root_path self._storage_list = [] self.CUR_DATE = 0 self.version_log = version_log self.menifest_xml = '' self.cp_version = None self.dsp_version = None self.log = MyLogger(self.__class__.__name__) self.verion_name = None def git_init(self): for storage in self._storage_list: _path = os.path.join(self.git_root_dir, storage) _repo = git.Repo(_path) _git = _repo.git try: # _git.config("--global", "user.name", "binwu") # _git.config("--global", "user.email", "*****@*****.**") _git.status() except Exception, e: self.log.error(e)
def __init__(self): super(CusRepo, self).__init__() self.log = MyLogger(self.__class__.__name__) self.git_clean() self.checkout_branch() self.log.info("create repo done")
class CraneExternalSRepo(RepoBase): def __init__(self): super(CraneExternalSRepo, self).__init__() self.log = MyLogger(self.__class__.__name__) self.log.info("create repo done") def get_config(self): json_file = os.path.join(self.root_dir, "json", "repo.json") json_str = load_json(json_file) self.config_d = json_str["crane_external_info"] self.branch_name = "master"
class craneCR2RCRepo(CusRepo): def __init__(self): super(craneCR2RCRepo, self).__init__() self.log = MyLogger(self.__class__.__name__) self.chip_ID = "craneC" self.log.info("create repo done") def get_config(self): json_file = os.path.join(self.root_dir, "json", "repo.json") json_str = load_json(json_file) self.config_d = json_str["cus_cranec_info"] self.branch_name = "r2_rc"
def __init__(self): super(DownloadToolController, self).__init__() self.root_dir = os.getcwd() self.download_tool = None self.log = MyLogger(self.__class__.__name__) self.download_tool_dict = {} self.release_download_tool_name = None self.download_tool_release_zip_dir = "" self.zip_tool = zipTool() self.update()
def main(): """ Scheduling :return: """ try: sched = BlockingScheduler(timezone='Asia/Kolkata') sched.add_job(mempool_crawler, 'interval', id='erc_mempool_crawler', seconds=3) sched.start() except Exception as e: obj_logger = MyLogger(logs_directory,category) obj_logger.error_logger('Main : %s'%(e))
class craneGRepo(RepoBase): def __init__(self): super(craneGRepo, self).__init__() self.log = MyLogger(self.__class__.__name__) self.chip_ID = "craneG" self.ps_mode = "LWG" self.log.info("create repo done") def get_config(self): json_file = os.path.join(self.root_dir, "json", "repo.json") json_str = load_json(json_file) self.config_d = json_str["craneg_info"] self.branch_name = "master"
class gitPushR2RCSDK(gitPushSDKBase): def __init__(self): super(gitPushR2RCSDK, self).__init__() self.log = MyLogger(self.__class__.__name__) self.branch_name = "r2_rc" # self.git.checkout(self.branch_name) def get_config(self): json_file = os.path.join(self.root_dir, "json", "git_push.json") json_str = load_json(json_file) self.config_d = json_str["cus_r2_rc_sdk"] def get_dsp_rf_dir(self, root_dir): for root, dirs, files in os.walk(root_dir, topdown=False): if "DSP" in dirs: self.dsp_rf_root_dir = os.path.join(root, "DSP") self.git_push_dsp_dir = os.path.dirname(self.target_dist_dir) dsp_bin = os.path.join(self.dsp_rf_root_dir, "CRANE", "CAT1GSM", "dsp.bin") rf_bin = os.path.join(self.dsp_rf_root_dir, "CRANE", "CAT1GSM", "rf.bin") crane_dsp_rf_dir = os.path.join(self.git_push_dsp_dir, "cus", "evb", "images") self.git_push_dsp_rf_list.append((dsp_bin, crane_dsp_rf_dir)) self.git_push_dsp_rf_list.append((rf_bin, crane_dsp_rf_dir)) dsp_bin = os.path.join(self.dsp_rf_root_dir, "CRANEG", "CAT1WCDMAGSM", "dsp.bin") rf_bin = os.path.join(self.dsp_rf_root_dir, "CRANEG", "CAT1WCDMAGSM", "rf.bin") craneg_a0_dir = os.path.join(self.git_push_dsp_dir, "cus", "evb_g_a0", "images") self.git_push_dsp_rf_list.append((dsp_bin, craneg_a0_dir)) self.git_push_dsp_rf_list.append((rf_bin, craneg_a0_dir)) dsp_bin = os.path.join(self.dsp_rf_root_dir, "CRANEG", "CAT1GSM", "dsp.bin") rf_bin = os.path.join(self.dsp_rf_root_dir, "CRANEG", "CAT1GSM", "rf.bin") cranec_dir = os.path.join(self.git_push_dsp_dir, "cus", "evb_c", "images") self.git_push_dsp_rf_list.append((dsp_bin, cranec_dir)) self.git_push_dsp_rf_list.append((rf_bin, cranec_dir)) self.log.info(self.git_push_dsp_rf_list) break
def __init__(self, version_log="", root_path=""): self.root_dir = os.getcwd() self.release_dist_dir = '' self.build_root_dir = '' self.git_root_dir = root_path self._storage_list = [] self.CUR_DATE = 0 self.version_log = version_log self.menifest_xml = '' self.cp_version = None self.dsp_version = None self.log = MyLogger(self.__class__.__name__) self.verion_name = None
def __init__(self): super(GitPushDspBase, self).__init__() self.log = MyLogger(self.__class__.__name__) self.decompress_tool = zipTool() self.release_dsp_bin = '' self.dsp_version = '' self.git_push_dsp_rf_list = [] self.rf_type = "PM813" self.update() self.create_git()
def __init__(self): super(RepoBase, self).__init__() super(myRepo, self).__init__() self.log = MyLogger(self.__class__.__name__) self.update() self.git_init()
def __init__(self): self.root_dir = os.getcwd() self.version_log = "" self.log = MyLogger(self.__class__.__name__) self.verion_name = '' self.cur_version = '' self.chip_ID = None self.OS_type = None self.ps_mode = None self.decompress_tool = zipTool() self.dsp_version_pattern = re.compile( r"(CRANE_.*?,.*?[0-9][0-9]:[0-9][0-9]:[0-9][0-9]|CRANEG_.*?,.*?[0-9][0-9]:[0-9][0-9]:[0-9][0-9])" ) self.version_pattern = None self.cp_version = None self.dsp_version = None
def __init__(self): super(gitPushSDKBase, self).__init__() self.log = MyLogger(self.__class__.__name__) self.zip_tool = zipTool() self.cp_sdk_version = None self.cp_sdk = None self.cp_sdk_root_dir = None self.dsp_rf_root_dir = '' self.git_push_dsp_dir = '' self.git_push_dsp_rf_list = [] self.dsp_bin = None self.rf_bin = None self.update() self.create_git()
def block_crawler(): """ Block Crawling process :return: """ obj_logger = MyLogger(logs_directory, category) obj_logger.msg_logger('Getting Block Numbers.....') # Get Current Block from RPC current_block = int( rpc_request(obj_logger, 'eth_blockNumber', []).get('result', 0), 16) crawled_blocks = int(redis_conn.get('eth_eth_blocks_crawled') or 0) obj_logger.msg_logger('Crawled Block Number : %s' % (crawled_blocks)) obj_logger.msg_logger('Current Block Number : %s' % (current_block)) obj_logger.msg_logger('Pending : %s' % (current_block - crawled_blocks)) if current_block > crawled_blocks: for block_number in range(crawled_blocks + 1, current_block + 1): obj_logger.msg_logger('#' * 100) obj_logger.msg_logger('Crawling Block : %s || Current Block : %s' % (block_number, current_block)) obj_logger.msg_logger('Pending : %s' % (current_block - block_number)) obj_logger.msg_logger('Start :%s' % (datetime.datetime.now())) # Increment Confirmations for tx_id whose 1 confirmation is already sent for tx_hash in redis_conn.smembers('eth_eth_pct_set'): tx_hash = tx_hash.decode('utf-8') data = find_sql_join( logger=obj_logger, table_names=[ 'user_master', 'address_master', 'eth_transactions' ], filters={'eth_transactions.tx_hash': tx_hash}, on_conditions={ 'user_master.user_name': 'address_master.user_name', 'address_master.address': 'eth_transactions.to_address' }, ) if not data: obj_logger.error_logger( '>>>>>>>>>>> Data not found in SQL for tx_hash : %s' % (tx_hash)) continue confirmations = data[0]['confirmations'] notif_url = data[0]['notification_url'] if confirmations < confirmation_threshold: increment_sql(obj_logger, 'eth_transactions', {'tx_hash': tx_hash}, 'confirmations') notif_params = { 'from_address': data[0]['from_address'], 'to_address': data[0]['to_address'], 'tx_hash': tx_hash, 'bid_id': -1, 'confirmations': confirmations + 1, 'block_number': data[0]['block_number'], 'value': data[0]['value'], 'flag': 'eth_incoming' } obj_logger.msg_logger( '>>>>>>>> Sending Confirmation : %s || %s' % (confirmations + 1, tx_hash)) send_notification(obj_logger, notif_url, notif_params, queue=hook_queue) else: obj_logger.msg_logger( '>>>>>>>> %s Confirmation Sent : %s' % (confirmation_threshold, tx_hash)) obj_logger.msg_logger( '>>>>>>>> Removing from eth_eth_pct_set : %s' % (tx_hash)) redis_conn.srem('eth_eth_pct_set', tx_hash) # Crawling Blocks block_info = rpc_request(obj_logger, 'eth_getBlockByNumber', [hex(int(block_number)), True]) if block_info: block_transactions = block_info.get('result', {}).get( 'transactions', []) else: block_transactions = [] obj_logger.error_logger( 'Data not found for block number : %s' % str(block_number)) for tx in block_transactions: to_address = tx['to'] if (redis_conn.sismember('eth_eth_aw_set', to_address)): tx_hash = tx['hash'] obj_logger.msg_logger( '>>>>>>>> Transaction Found in Block : %s : %s' % (block_number, tx_hash)) confirmations = 1 block_number = int(tx['blockNumber'], 16) # Check if 1 Confirmation is sent from mempool crawler - Should be found in eth_eth_pct_set if not redis_conn.sismember('eth_eth_pct_set', tx_hash): from_address = tx['from'] value = int(tx['value'], 16) bid_id = -1 flag = 'eth_incoming' sys_timestamp = datetime.datetime.now() # Check if 0 Confirmation is sent from mempool crawler - Should be found in eth_eth_zct_set if redis_conn.sismember('eth_eth_zct_set', tx_hash): update_sql(obj_logger, 'eth_transactions', {'tx_hash': tx_hash}, updated_values={ 'confirmations': confirmations, 'block_number': block_number }) else: # Missed in Mempool - Send 1 Confirmation and add in eth_eth_pct_set obj_logger.msg_logger( '>>>>>>>> Transaction Missed from mempool. Sending %s confirmation : %s' % (confirmations, str(tx_hash))) data = { 'from_address': from_address, 'to_address': to_address, 'tx_hash': tx_hash, 'bid_id': bid_id, 'confirmations': confirmations, 'block_number': block_number, 'value': value, 'flag': flag, 'sys_timestamp': sys_timestamp, } insert_sql(obj_logger, 'eth_transactions', data) notif_url = find_sql_join( logger=obj_logger, table_names=['user_master', 'address_master'], filters={'address_master.address': to_address}, on_conditions={ 'user_master.user_name': 'address_master.user_name' }, columns=['user_master.notification_url' ])[0]['notification_url'] notif_params = { 'from_address': from_address, 'to_address': to_address, 'tx_hash': tx_hash, 'bid_id': -1, 'confirmations': confirmations, 'block_number': block_number, 'value': value, 'flag': flag } obj_logger.msg_logger( '>>>>>>>> Sending Confirmation : %s || %s' % (confirmations, tx_hash)) send_notification(obj_logger, notif_url, notif_params, queue=hook_queue) obj_logger.msg_logger( '>>>>>>>> Adding to eth_eth_pct_set : %s' % (tx_hash)) redis_conn.sadd('eth_eth_pct_set', tx_hash.encode('utf-8')) # Increment Redis Blocks Crawled redis_conn.set('eth_eth_blocks_crawled', block_number) obj_logger.msg_logger('Ends :%s' % (datetime.datetime.now())) obj_logger.msg_logger('#' * 100) else: obj_logger.msg_logger('#' * 100)
class DownloadToolController(object): def __init__(self): super(DownloadToolController, self).__init__() self.root_dir = os.getcwd() self.download_tool = None self.log = MyLogger(self.__class__.__name__) self.download_tool_dict = {} self.release_download_tool_name = None self.download_tool_release_zip_dir = "" self.zip_tool = zipTool() self.update() def update(self): json_file = os.path.join(self.root_dir,"json","download_tool.json") json_str = load_json(json_file) self.tmp = os.path.join(self.root_dir, json_str["tmp"]) self.download_tool_dir = os.path.join(self.root_dir, json_str["download_tool_dir"]) self.partition_config = os.path.join(self.root_dir, json_str["partition_config"]) self.template_config = os.path.join(self.root_dir, json_str["template_config"]) self.download_tool_release_dir = json_str["download_tool_release_dir"] self.download_tool_file_name = json_str["download_tool_file_name"] self.win_type = json_str["win_type"] def unzip_download_tool(self,zip_file,extract_dir=None): self.zip_tool.unpack_archive(os.path.join(self.download_tool_dir,zip_file),extract_dir) def update_download_tool(self): tools_list = [_file for _file in os.listdir(self.download_tool_release_dir) if self.win_type in _file and "aboot-tools" in _file and _file.endswith(".exe")] tools_list.sort(key=lambda fn: os.path.getmtime(self.download_tool_release_dir + '\\' + fn)) self.log.debug(tools_list) if not self.download_tool_file_name: self.download_tool_file_name = tools_list[-1] file_name = self.download_tool_file_name self.download_tool_release_zip_dir = os.path.join(self.download_tool_dir, self.download_tool_file_name.replace(".exe","")) # self.log.info(file_name) file_name_l = [file_name,file_name.replace("x64","x86")] self.download_tool = [] for file_name in file_name_l: zip_file = os.path.join(self.download_tool_dir,file_name) # self.log.info(zip_file) download_tool_fname = zip_file.replace(".exe","") if not os.path.exists(zip_file): shutil.copy2(os.path.join(self.download_tool_release_dir,file_name),self.download_tool_dir) time.sleep(3) self.unzip_download_tool(zip_file) elif not os.path.exists(download_tool_fname): time.sleep(3) self.unzip_download_tool(zip_file) self.download_tool.append(download_tool_fname) for _tool in self.download_tool: self.log.info(_tool) def prepare_download_tool(self,images,borad="crane_evb_z2"): """borad : crane_evb_z2, bird_phone, crane_evb_dual_sim""" self.log.debug("\n".join(images)) self.log.debug("\n".join(self.download_tool)) for download_tool_dir in self.download_tool: if not os.path.exists(download_tool_dir): self.log.warning("%s not exists" % download_tool_dir) continue dist_dir = os.path.join(download_tool_dir,"images") dist_bin_l = [os.path.join(dist_dir,os.path.basename(_file)) for _file in images] for _file in dist_bin_l: if os.path.exists(_file): os.remove(_file) for src_bin,dist_bin in zip(images,dist_bin_l): if os.path.exists(src_bin): if os.path.exists(dist_bin): os.remove(dist_bin) shutil.copy2(src_bin,dist_bin) else: self.log.warning("%s not exists" % src_bin) if os.path.isdir(self.partition_config): for _file in os.listdir(self.partition_config): shutil.copy2(os.path.join(self.partition_config,_file), os.path.join(download_tool_dir,"config","partition",_file)) elif os.path.isfile(self.partition_config): shutil.copy2(self.partition_config,os.path.join(download_tool_dir,"config","partition", os.path.basename(self.partition_config))) else: self.log.error("self.partition_config:%s error" % self.partition_config) if os.path.isdir(self.template_config): for _file in os.listdir(self.template_config): shutil.copy2(os.path.join(self.template_config,_file), os.path.join(download_tool_dir,"config","template",_file)) elif os.path.isfile(self.template_config): shutil.copy2(self.template_config, os.path.join(download_tool_dir,"config","template",os.path.basename(self.template_config))) else: self.log.error("self.template_config:%s error" % self.template_config) def release_zip(self, dist_dir, project_type = "CRANE_A0_16MB", zip_name = None): assert os.path.exists( self.download_tool_release_zip_dir)," can not find download tool %s " % self.download_tool_release_zip_dir os.chdir(self.download_tool_release_zip_dir) if "craneg_evb" in dist_dir: project_type = "CRANEG_Z1_32+8MB" if zip_name: zip_file = zip_name else: zip_file = "ASR_CRANE_EVB_%s.zip" % project_type zip_file = os.path.join(dist_dir,zip_file) release_log = os.path.join(self.tmp,"release_log.txt") self.log.info("create zip %s" % zip_file) release_cmd = "arelease.exe -c . -g -p ASR_CRANE_EVB -v %s %s > %s" % ( project_type, zip_file, release_log) os.system(release_cmd) def release_download_tool(self,release_name,borad="crane_evb_z2",dist_dir=None, download_tool_l=[]): import time "borad : crane_evb_z2, bird_phone, crane_evb_dual_sim" date = time.strftime("%Y%m%d_%H%M%S") release_file_name = "%s_%s_DOWNLOAD_TOOL" % (release_name.upper(),borad.upper()) release_dir = os.path.join(self.download_tool_dir,release_file_name) os.mkdir(release_dir) if not os.path.exists(release_dir) else None self.log.info(release_file_name) # self.log.info(release_dir) if download_tool_l: download_tool_src = download_tool_l[:] else: download_tool_src = self.download_tool for _tool in download_tool_src: dist_file = os.path.join(release_dir,os.path.basename(_tool)) # self.log.info(dist_file) if not os.path.exists(dist_file): shutil.copytree(_tool,dist_file) dist = os.path.join(dist_dir,release_file_name) self.zip_tool.make_archive_e(dist,"zip",release_dir) shutil.rmtree(release_dir) self.release_download_tool_name = dist + ".zip" self.download_tool_dict[borad] = self.release_download_tool_name # self.log.info(self.download_tool_dict) return self.release_download_tool_name
def mempool_crawler(): """ Mempool Process :return: """ obj_logger = MyLogger(logs_directory, category) obj_logger.msg_logger('#' * 100) obj_logger.msg_logger('Getting Mempool Data') # Get Mempool Data mempool_transaction_data = rpc_request(obj_logger, 'eth_getBlockByNumber', ['pending', True]).get( 'result', {}).get('transactions', []) obj_logger.msg_logger('Crawling Mempool Starts') for tx in mempool_transaction_data: tx_hash = tx['hash'] to_address = tx['to'] # Redis Check if (not redis_conn.sismember('eth_eth_zct_set', tx_hash)) and ( redis_conn.sismember('eth_eth_aw_set', to_address)): obj_logger.msg_logger( '>>>>>>>> Transaction Found in Mempool : %s' % (tx_hash)) from_address = tx['from'] value = int(tx['value'], 16) bid_id = -2 confirmations = 0 block_number = -1 flag = 'eth_incoming' sys_timestamp = datetime.datetime.now() # Insert in DB result = insert_sql(logger=obj_logger, table_name='eth_transactions', data={ 'from_address': from_address, 'to_address': to_address, 'tx_hash': tx_hash, 'bid_id': bid_id, 'confirmations': confirmations, 'block_number': block_number, 'value': value, 'flag': flag, 'sys_timestamp': sys_timestamp, }) if result: notif_url = find_sql_join( logger=obj_logger, table_names=['user_master', 'address_master'], filters={'address_master.address': to_address}, on_conditions={ 'user_master.user_name': 'address_master.user_name' }, columns=['user_master.notification_url' ])[0]['notification_url'] notif_params = { 'from_address': from_address, 'to_address': to_address, 'tx_hash': tx_hash, 'bid_id': bid_id, 'confirmations': confirmations, 'block_number': block_number, 'value': value, 'flag': flag } send_notification(obj_logger, notif_url, notif_params, queue=hook_queue) obj_logger.msg_logger( '>>>>>>>> Adding to eth_eth_zct_set : %s' % (tx_hash)) redis_conn.sadd( 'eth_eth_zct_set', tx_hash.encode('utf-8') ) # To cross check in Block Crawler and not to send multiple notification obj_logger.msg_logger('Crawling Mempool Ends') obj_logger.msg_logger('#' * 100)
def __init__(self): super(gitPushR2RCSDK, self).__init__() self.log = MyLogger(self.__class__.__name__) self.branch_name = "master"
def __init__(self): super(gitPushR2RCSDK008, self).__init__() self.log = MyLogger(self.__class__.__name__) self.branch_name = "r2_rc_sdk_1.008"
def __init__(self): super(gitPushCraneMDsp, self).__init__() self.log = MyLogger(self.__class__.__name__) self.rf_type = "PM803"
def __init__(self): super(cusR1Repo, self).__init__() self.log = MyLogger(self.__class__.__name__)
class gitPushSDKBase(GitPushBase): def __init__(self): super(gitPushSDKBase, self).__init__() self.log = MyLogger(self.__class__.__name__) self.zip_tool = zipTool() self.cp_sdk_version = None self.cp_sdk = None self.cp_sdk_root_dir = None self.dsp_rf_root_dir = '' self.git_push_dsp_dir = '' self.git_push_dsp_rf_list = [] self.dsp_bin = None self.rf_bin = None self.update() self.create_git() # self.print_info() def get_config(self): self.config_d = {} def update(self): self.get_config() self.cp_sdk_release_dir = os.path.normpath( self.config_d["release_dir"]) self.git_push_cp_dir = os.path.normpath( self.config_d["git_push_root_dir"]) self.git_push_root_dir = self.git_push_cp_dir self.target_dist_dir = os.path.normpath(self.config_d["target_dir"]) self.cp_sdk_dir = os.path.normpath( os.path.join(self.root_dir, self.config_d["local_dir"])) self.cp_version_file = self.config_d["verson_file"] self.release_target = self.config_d["release_target"] self.push_cmd = self.config_d["git_push_cmd"] def get_dsp_rf_dir(self, root_dir): for root, dirs, files in os.walk(root_dir, topdown=False): if "DSP" in dirs: self.dsp_rf_root_dir = os.path.join(root, "DSP") self.dsp_bin = os.path.join(self.dsp_rf_root_dir, "dsp.bin") self.rf_bin = os.path.join(self.dsp_rf_root_dir, "rf.bin") self.git_push_dsp_dir = os.path.dirname(self.target_dist_dir) self.git_push_dsp_dir = os.path.join(self.git_push_dsp_dir, "cus", "evb", "images") self.git_push_dsp_rf_list.append( (self.dsp_bin, self.git_push_dsp_dir)) self.git_push_dsp_rf_list.append( (self.rf_bin, self.git_push_dsp_dir)) break def find_new_cp_sdk(self): "ASR3603_MINIGUI_20200603_SDK.zip" cp_sdk_list = [ _file for _file in os.listdir(self.cp_sdk_release_dir) if (_file.endswith(".zip") or _file.endswith(".7z")) and _file.startswith(self.release_target) ] cp_sdk_list.sort(key=lambda fn: os.path.getmtime( os.path.join(self.cp_sdk_release_dir, fn))) assert cp_sdk_list, "can not find sdk" self.cp_sdk = cp_sdk_list[-1] self.log.debug("newest adk: %s" % self.cp_sdk) def clean_git_push_cp(self): self.git_clean() for _file in os.listdir(self.target_dist_dir): if _file in [".git", "X.bat"]: continue _file = os.path.join(self.target_dist_dir, _file) if os.path.isfile(_file): os.remove(_file) else: shutil.rmtree(_file) self.log.info("clean cp done") def copy_sdk(self): self.log.info("copy %s..." % self.cp_sdk) shutil.copy2(os.path.join(self.cp_sdk_release_dir, self.cp_sdk), self.cp_sdk_dir) time.sleep(3) self.log.info("copy done.") def copy_sdk_to_git_push_cp(self, cp_sdk): try: root_dir = os.path.join(self.cp_sdk_dir, cp_sdk) for _file in os.listdir(root_dir): fname = os.path.join(root_dir, _file) if os.path.isfile(fname): shutil.copy2(fname, os.path.join(self.target_dist_dir, _file)) elif os.path.isdir(fname): shutil.copytree(fname, os.path.join(self.target_dist_dir, _file)) else: self.log.warning("%s" % fname) self.log.info("%s" % self.dsp_rf_root_dir) if os.path.exists(self.dsp_rf_root_dir): dir_path = os.path.dirname(self.dsp_rf_root_dir) self.log.info("%s" % dir_path) dist_dir = os.path.join(self.target_dist_dir, os.path.basename(dir_path)) self.log.info("%s" % dist_dir) if os.path.exists(dist_dir): shutil.rmtree(dist_dir) shutil.copytree(dir_path, dist_dir) while self.git_push_dsp_rf_list: src_file, dist = self.git_push_dsp_rf_list.pop() self.log.info(src_file) self.log.info(dist) if os.path.exists(dist): if os.path.isfile(src_file): shutil.copy2(src_file, dist) else: self.log.warning("%s" % src_file) self.log.info("copy_sdk_to_git_push_cp done.") except Exception, e: self.log.error(e) self.log.error("copy_sdk_to_git_push_cp error") raise Exception, "copy_sdk_to_git_push_cp error"
class GitPushDspBase(GitPushBase): def __init__(self): super(GitPushDspBase, self).__init__() self.log = MyLogger(self.__class__.__name__) self.decompress_tool = zipTool() self.release_dsp_bin = '' self.dsp_version = '' self.git_push_dsp_rf_list = [] self.rf_type = "PM813" self.update() self.create_git() # self.print_info() def get_config(self): self.config_d = {} def update(self): self.get_config() self.release_dir = os.path.normpath(self.config_d["release_dir"]) self.target_dist_dir = os.path.normpath(self.config_d["target_dir"]) self.git_push_root_dir = os.path.normpath( self.config_d["git_push_root_dir"]) self.local_dsp_bin = os.path.normpath(self.config_d["verson_file"]) self.push_cmd = self.config_d["git_push_cmd"] self.dsp_version_pattern = re.compile(self.config_d["version_pattern"]) self.release_target_file = self.config_d["release_target"] def update_git_push_dsp_rf_list(self): root_dir = os.path.dirname(self.release_dsp_bin) self.git_push_dsp_rf_list.append( (self.release_dsp_bin, self.local_dsp_bin)) rf_bin = os.path.join(root_dir, self.rf_type, "rf.bin") self.git_push_dsp_rf_list.append((rf_bin, self.target_dist_dir)) rf_verson_file = os.path.join(root_dir, self.rf_type, "RF_Version.txt") self.git_push_dsp_rf_list.append( (rf_verson_file, self.target_dist_dir)) rf_excel_file = os.path.join(root_dir, self.rf_type, "rf.xlsm") self.git_push_dsp_rf_list.append((rf_excel_file, self.target_dist_dir)) def get_dsp_version(self, dsp_bin=None): """CRANE_CAT1GSM_L1_1.043.000 , Dec 13 2019 03:30:56""" if not dsp_bin: dsp_bin = self.local_dsp_bin dsp_version_file = os.path.join(self.root_dir, "tmp", "dsp_version_tmp.bin") self.decompress_tool.decompress_bin(dsp_bin, dsp_version_file) assert os.path.exists(dsp_version_file), "can not find {}".format( dsp_version_file) with open(dsp_version_file, "rb") as fob: text = fob.read() match = self.dsp_version_pattern.findall(text) if match: self.log.debug(match[0]) version_info = match[0] else: self.log.error("can not find dsp version infomation") version_info = None # os.remove(dsp_version_file) return version_info def get_release_dsp_rf(self): dsp_release_bin_l = [] release_dir_list = [os.path.join(self.release_dir,_dir) for _dir in os.listdir(self.release_dir) \ if os.path.isdir(os.path.join(self.release_dir,_dir))] release_dir_list.sort(key=lambda fn: os.path.getmtime(fn)) self.log.debug("release_dir_list len:", len(release_dir_list)) self.log.debug(release_dir_list[-10:]) release_files = [ os.path.normpath(os.path.join(_dir, self.release_target_file)) for _dir in release_dir_list[-10:] ] for _file in release_files: if os.path.exists(_file): root = os.path.dirname(_file) rf = os.path.join(root, self.rf_type, "rf.bin") if os.path.exists(rf): dsp_release_bin_l.append(_file) dsp_release_bin_l.sort(key=lambda fn: os.path.getmtime(fn)) # self.log.info("\n".join(dsp_release_bin_l)) self.release_dsp_bin = dsp_release_bin_l[-1] def condition(self): self.update() self.git_clean() local_dsp_version = self.get_dsp_version() self.get_release_dsp_rf() release_dsp_version = self.get_dsp_version(self.release_dsp_bin) if local_dsp_version == release_dsp_version: return False else: self.log.info("local_dsp_version :", local_dsp_version) self.log.info("release_dsp_version:", release_dsp_version) self.dsp_version = release_dsp_version return True def close_push(self): to_address = ",".join(["*****@*****.**"]) subject = "%s auto push" % self.dsp_version msg = r"Hi %s, %s auto push done!" % (to_address.split("@")[0], self.dsp_version) send_email_tool(to_address, subject.upper(), msg) def git_push_start(self): if not self.condition(): time.sleep(10) return self.log.info("wait for dsp copy...") time.sleep(30) self.update_git_push_dsp_rf_list() while self.git_push_dsp_rf_list: src_file, dist = self.git_push_dsp_rf_list.pop() self.log.info(src_file) self.log.info(dist) if os.path.exists(dist): if os.path.isfile(src_file): shutil.copy2(src_file, dist) else: self.log.warning("%s" % src_file) self.log.info("=" * 50) self.log.info("git push dsp...") try: self.git_add() match = re.findall(self.dsp_version_pattern, self.dsp_version) if match: dsp_version = match[0] else: dsp_version = str(time.asctime()) commit_info = "update dsp dailybuild %s" % dsp_version self.git_commit(commit_info) self.git_push() self.close_push() return True except Exception, e: self.log.error(e) self.log.error("git push error") self.git_clean() return None
def __init__(self): super(gitPushCraneFTR2RCSDK, self).__init__() self.log = MyLogger(self.__class__.__name__) self.branch_name = "crane_ft"
class GitPushBase(object): def __init__(self): self.root_dir = os.getcwd() self.log = MyLogger(self.__class__.__name__) self.git_push_root_dir = None self.branch_name = "master" def create_git(self): self.git = git.Repo(self.git_push_root_dir).git # self.git.config("--global","core.autocrlf","false") # self.git.config("--global","user.name","binwu") # self.git.config("--global","user.email","*****@*****.**") def git_add(self, *file_name_l): self.log.info("git add...") if file_name_l: self.git.add(*file_name_l) else: self.git.add("--all") self.log.info("git add done") def git_commit(self, commit_info): self.log.info("git commit...") self.log.info("conmmit info:", commit_info) self.git.commit("-m %s" % commit_info) self.log.info("git commit done") def git_push(self): self.log.info("git push...") self.git.push(*self.push_cmd) self.log.info("git push done") def git_clean(self): try: self.git.clean("-xdf") self.git.reset("--hard", "origin/%s" % self.branch_name) self.git.pull() except Exception, e: self.log.error(e) self.git.reset("--hard", "origin/%s" % self.branch_name) raise Exception("git_clean error")
def callback(ch, method, properties, body): """ This method is called every time there is a new element in queue (var : queue) :param ch: :param method: :param properties: :param body: :return: """ try: # Logger obj_logger = MyLogger(logs_directory, category) obj_logger.msg_logger('#'*100) obj_logger.msg_logger('In Exception Queue : %s'%(queue)) obj_logger.msg_logger('Getting Data : %s'%(datetime.datetime.now())) # Data from Queue data = json.loads(body) notification_url = data['notification_url'] data.pop('notification_url') notification_params = data obj_logger.msg_logger('>>>>>>>>>> Sending Notification : %s || %s' % (notification_url, notification_params)) # Send Notification requests.post(notification_url, data=json.dumps(notification_params), headers=headers) obj_logger.msg_logger('>>>>>>>>>> Notification Success : %s || %s' % (notification_url, notification_params)) # Insert in DB insert_sql(logger=obj_logger, table_name='notification_logs', data={ 'tx_hash': notification_params['tx_hash'], 'notification_url ': notification_url, 'params': str(notification_params), 'timestamp': datetime.datetime.now(), 'Status': 'Success' }) ch.basic_ack(delivery_tag=method.delivery_tag) except Exception as e: obj_logger.error_logger('>>>>>>>>>> Notification Failure : %s || %s || %s' % (e, notification_url, notification_params)) finally: obj_logger.msg_logger("#" * 100)
def __init__(self): super(gitPushCraneDCXODsp, self).__init__() self.log = MyLogger(self.__class__.__name__)
def callback(ch, method, properties, body): """ This method is called every time there is a new element in queue (var : queue) :param ch: :param method: :param properties: :param body: :return: """ try: # Logs obj_logger = MyLogger(logs_directory, category) obj_logger.msg_logger('#' * 100) obj_logger.msg_logger('Getting Data : %s' % (datetime.datetime.now())) # Data from Queue data = json.loads(body) notification_url = data['notification_url'] data.pop('notification_url') notification_params = data obj_logger.msg_logger('>>>>>>>>>> Sending Notification : %s || %s' % (notification_url, notification_params)) # Send Notification requests.post(notification_url, data=json.dumps(notification_params), headers=headers) obj_logger.msg_logger('>>>>>>>>>> Notification Success : %s || %s' % (notification_url, notification_params)) # Insert in DB insert_sql(logger=obj_logger, table_name='notification_logs', data={ 'tx_hash': notification_params['tx_hash'], 'notification_url ': notification_url, 'params': str(notification_params), 'timestamp': datetime.datetime.now(), 'Status': 'Success' }) except Exception as e: # If there is an Exception , Send the Notification to Exception Queue - which will be handled manually obj_logger.error_logger( '>>>>>>>>>> Notification Failure : %s || %s || %s' % (e, notification_url, notification_params)) obj_logger.msg_logger('>>>>>>>>>> Pushing to Exception Queue : %s' % (exception_queue)) send_notification(obj_logger, notification_url, notification_params, queue=exception_queue) finally: obj_logger.msg_logger("#" * 100) # We are ACK in both the case of success or failure because if there is no error then its ok # But if there is an error then we are sending it to Exception Queue . So in both the case we can delete this from main queue ch.basic_ack(delivery_tag=method.delivery_tag)
class GitPushDMSDK(gitPushR2RCSDK): def __init__(self): super(gitPushR2RCSDK, self).__init__() self.log = MyLogger(self.__class__.__name__) self.branch_name = "master" # self.git.checkout(self.branch_name) def get_config(self): json_file = os.path.join(self.root_dir, "json", "git_push.json") json_str = load_json(json_file) self.config_d = json_str["dm_sdk"] def get_dsp_rf_dir(self, root_dir): self.git_push_dsp_dir = os.path.dirname(self.target_dist_dir) for root, dirs, files in os.walk(root_dir, topdown=False): if "DSP" in dirs: self.dsp_rf_root_dir = os.path.join(root, "DSP") dsp_bin = os.path.join(self.dsp_rf_root_dir, "CRANEM", "CAT1GSM", "dsp.bin") rf_bin = os.path.join(self.dsp_rf_root_dir, "CRANEM", "CAT1GSM", "PM803", "rf.bin") cranem_dm_dir = os.path.join(self.git_push_dsp_dir, "cus", "evb_dm", "images") self.git_push_dsp_rf_list.append((dsp_bin, cranem_dm_dir)) self.git_push_dsp_rf_list.append((rf_bin, cranem_dm_dir)) if "CP" in dirs: cp_root_dir = os.path.join(root, "CP") boot33_bin = os.path.join(cp_root_dir, "CAT1GSM", "boot33.bin") apn_bin = os.path.join(cp_root_dir, "CAT1GSM", "apn.bin") cranem_dm_dir = os.path.join(self.git_push_dsp_dir, "cus", "evb_dm", "images") self.git_push_dsp_rf_list.append((boot33_bin, cranem_dm_dir)) self.git_push_dsp_rf_list.append((apn_bin, cranem_dm_dir)) break cranem_dm_dir = os.path.join(self.git_push_dsp_dir, "cus", "evb_dm", "build") Crane_DS_miniPhone_h = os.path.join( cranem_dm_dir, "Crane_DS_miniPhone_16M_Ram_8M_Flash_XIP_CIPSRAM.h") Crane_DS_miniPhone_sct = os.path.join( cranem_dm_dir, "Crane_DS_miniPhone_16M_Ram_8M_Flash_XIP_CIPSRAM.sct") dist_dir = os.path.join(self.target_dist_dir, "csw", "platform", "dev_plat", "build") self.git_push_dsp_rf_list.append((Crane_DS_miniPhone_h, dist_dir)) self.git_push_dsp_rf_list.append((Crane_DS_miniPhone_sct, dist_dir)) for _item in self.git_push_dsp_rf_list: self.log.info(_item) def find_new_cp_sdk(self): cp_sdk_list = [ _file for _file in os.listdir(self.cp_sdk_release_dir) if _file.startswith(self.release_target) ] cp_sdk_list.sort(key=lambda fn: os.path.getmtime( os.path.join(self.cp_sdk_release_dir, fn))) assert cp_sdk_list, "can not find sdk" self.cp_sdk = cp_sdk_list[-1] self.log.debug("newest adk: %s" % self.cp_sdk) def copy_sdk(self): copy(os.path.join(self.cp_sdk_release_dir, self.cp_sdk), os.path.join(self.cp_sdk_dir, self.cp_sdk)) time.sleep(3) def unzip_sdk(self): fname, _ = os.path.splitext(self.cp_sdk) root_dir = os.path.join(self.cp_sdk_dir, self.cp_sdk, "SDK") sdk_zip = [ os.path.join(root_dir, _file) for _file in os.listdir(root_dir) if _file.startswith(self.release_target) ] assert sdk_zip, "can not find %s" % root_dir sdk_zip = sdk_zip[-1] while True: try: with ziptool_mutex: self.zip_tool.unpack_archive(sdk_zip, root_dir) assert os.path.exists(root_dir), "can not find %s" % root_dir for root, dirs, files in os.walk(root_dir, topdown=False): if "3g_ps" in dirs: self.cp_sdk_root_dir = root break assert os.path.exists( self.cp_sdk_root_dir ), "can not find %s" % self.cp_sdk_root_dir self.get_dsp_rf_dir(os.path.join(self.cp_sdk_dir, self.cp_sdk)) return except Exception, e: time.sleep(10) self.log.error(e)
def mempool_crawler(): """ Mempool Process :return: """ obj_logger = MyLogger(logs_directory,category) obj_logger.msg_logger('#'*100) obj_logger.msg_logger('Getting Mempool Data') # Get Mempool Data mempool_transaction_data = rpc_request(obj_logger, 'eth_getBlockByNumber', ['pending', True]).get('result',{}).get('transactions',[]) obj_logger.msg_logger('Crawling Mempool Starts') for tx in mempool_transaction_data: tx_hash = tx['hash'] contract_address = tx['to'] # To address in ERC 20 Transaction is Contract Address # TODO - if tx hashes are not matching in redis, then we need to encode/decode utf-8 # Redis Check if (not redis_conn.sismember('eth_erc_zct_set',tx_hash)) and (redis_conn.sismember('eth_erc_aw_set',contract_address)): obj_logger.msg_logger('>>>>>>>> Transaction Found in Mempool : %s'%(tx_hash)) from_address = tx['from'] bid_id = -1 confirmations = 0 block_number = -1 flag = 'erc20' sys_timestamp = datetime.datetime.now() # Decoding Inputs input = tx['input'] with open(abi_file, 'r') as abi_definition: abi = json.load(abi_definition) contract_obj = web3.Web3().eth.contract(address=web3.Web3().toChecksumAddress(contract_address), abi=abi) params = contract_obj.decode_function_input(input) to_address = params[1].get('_to') value = params[1].get('_value') # Insert in DB result = insert_sql( logger=obj_logger, table_name= 'erc_transactions', data={ 'from_address': from_address, 'to_address': to_address, 'contract_address': contract_address, 'tx_hash': tx_hash, 'bid_id': bid_id, 'confirmations': confirmations, 'block_number': block_number, 'value': value, 'flag': flag, 'sys_timestamp': sys_timestamp, } ) if result: notif_url = find_sql_join(logger=obj_logger, table_names=['user_master', 'erc_address_master'], filters={'erc_address_master.address': to_address}, on_conditions={'user_master.user_name': 'erc_address_master.user_name'}, columns=['user_master.notification_url'] )[0]['notification_url'] notif_params = { 'from_address': from_address, 'to_address': to_address, 'contract_address': contract_address, 'tx_hash': tx_hash, 'bid_id': bid_id, 'confirmations': confirmations, 'block_number': block_number, 'value': value, 'flag': flag } send_notification(obj_logger,notif_url,notif_params,queue=hook_main) obj_logger.msg_logger('>>>>>>>> Adding to eth_erc_zct_set : %s' % (tx_hash)) redis_conn.sadd('eth_erc_zct_set', tx_hash.encode('utf-8')) # To cross check in Block Crawler and not to send multiple notification obj_logger.msg_logger('Crawling Mempool Ends') obj_logger.msg_logger('#' * 100)
def __init__(self): self.root_dir = os.getcwd() self.log = MyLogger(self.__class__.__name__) self.git_push_root_dir = None self.branch_name = "master"