def ListTimeZones(self): """Lists the timezones.""" max_length = 0 for timezone_name in pytz.all_timezones: if len(timezone_name) > max_length: max_length = len(timezone_name) utc_date_time = datetime.datetime.utcnow() print('{0:20}\t{1:s}'.format('TimezoneUTC', 'Offset')) for timezone_name in pytz.all_timezones: try: local_timezone = pytz.timezone(timezone_name) except AssertionError as exception: logger.error(( 'Unable to determine information about timezone: {0:s} with ' 'error: {1!s}').format(timezone_name, exception)) continue local_date_string = '{0!s}'.format( local_timezone.localize(utc_date_time)) if '+' in local_date_string: _, _, diff = local_date_string.rpartition('+') diff_string = '+{0:s}'.format(diff) else: _, _, diff = local_date_string.rpartition('-') diff_string = '-{0:s}'.format(diff) print('{0:20}\t{1:s}'.format(timezone_name, diff_string))
def pull_feeds(url, provider, rss_category): """ Pull feeds for an individual category specific Article :param url: Category based Provider URL link :param provider: Name of the Article Provider :param rss_category: Category of the articles :return: None """ try: # Get provider from the config file items = parse_url(url) if items: # Loop through the items iterator object for item in items: # if no title or description skip the article if not item['title'] and item['description']: continue # Update Metadata result = update_metadata(item, rss_category, provider) # Update Mongo DB update_mongo(result, provider) else: logger.warning( "Unable to pull items for `{}` in `{}` category".format( provider, rss_category)) except Exception as error: logger.error(error)
def checkFabricServiceStatus(yamlContent): logger.info("Waiting for BoK to start up ...") clusterName = yamlContent["clusterName"] peerOrgs = yamlContent["crypto-config.yaml"]["PeerOrgs"] ifReady=0 timeCount=0 config.load_kube_config() v1 = client.CoreV1Api() while True: ret = v1.list_pod_for_all_namespaces(watch=False) for peerOrg in peerOrgs: peerOrgName=peerOrg["Name"] namespace=peerOrgName.lower() + "-" + clusterName.lower() for i in ret.items: if namespace == i.metadata.namespace: if i.status.phase != "Running": ifReady=0 break else: ifReady=1 if ifReady == 0: break if ifReady == 1: logger.info("BoK is up and running.") break else: timeCount = timeCount + INTERVAL time.sleep(INTERVAL) if timeCount > TIMEOUT: logger.error("Error: Failed to start BoK service.") sys.exit(1)
def get_metadata_newspaper(item): """ Use newspaper3k library to get more details like Image URL and other metadata :param item: Article data :return: item """ try: if not item['link'] and not item['id']: return {} if item['link'] and item['link'] != "": article = Article(item['link']) else: article = Article(item['id']) try: article.download() article.parse() except ArticleException: try: article = Article(item['link']) article.download() article.parse() except: article = {} if article: if article.title: item['title'] = article.title if article.text: item['description'] = article.text if article.top_image: item['image_url'] = article.top_image except Exception as error: logger.error(error) return item
def checkAndRun(f): if os.path.isfile(f): # sleep 5 seconds before deployment creation, # in case the PV hasn't initialized completely. os.system("kubectl apply -f " + f) else: logger.error("The file %s does not exist." % (f))
def CreateStorageAndTempPath(self, cursor, case_id=None, evd_id=None): if not case_id or not evd_id: print("ERROR: Missing Case ID or Evidence ID.") return if self._source_path is None: query = 'SELECT evd_path FROM evidence_info WHERE case_id="' + case_id + '" AND evd_id="' + evd_id + '";' try: evd_path = cursor.execute_query(query) if evd_path is None: raise ValueError self._source_path = os.path.join(self._root_storage_path, evd_path[0]) except ValueError: logger.error('Evidence is not exist: \'evd_path\' is None') raise Exception('Evidence is not exist: \'evd_path\' is None') except Exception as exception: logger.error( 'database execution failed: {0!s}'.format(exception)) raise Exception( 'database execution failed: {0!s}'.format(exception)) if not os.path.isdir(os.path.join(self._root_tmp_path, case_id)): os.mkdir(os.path.join(self._root_tmp_path, case_id)) self._tmp_path = os.path.join( os.path.join(self._root_tmp_path, case_id), evd_id) if not os.path.isdir(self._tmp_path): os.mkdir(self._tmp_path)
def pull_feeds(): try: pull_rss_feedparser.fetch() return jsonify({"Finished pulling all data!!!"}) except Exception as error: logger.error(error)
def Read(self): """Reads a string from the input. Returns: str: input. """ encoded_string = self._file_object.readline() if isinstance(encoded_string, str): return encoded_string try: string = codecs.decode(encoded_string, self._encoding, self._errors) except UnicodeDecodeError: if self._errors == 'strict': logger.error( 'Unable to properly read input due to encoding error. ' 'Switching to error tolerant encoding which can result in ' 'non Basic Latin (C0) characters to be replaced with "?" or ' '"\\ufffd".') self._errors = 'replace' string = codecs.decode(encoded_string, self._encoding, self._errors) return string
def parse_url(url): """ Parse RSS XML URL and get the basic details :param url: RSS XML URL :param provider: Name of the Article provider :return: item """ try: items = [] column_list = [ 'title', 'summary', 'id', 'language', 'link', 'description', 'published', 'media_content', 'image' ] elements = feedparser.parse(url) elements = (element for element in elements.entries) if not elements: return [] logger.info("Please wait, parsing through the RSS feeds........") for element_res in elements: item = {} for element in element_res: if element in column_list: item[element] = element_res.get(element) if 'media_content' in element_res: item['image_url'] = element_res.media_content[0]['url'] else: item['image_url'] = element_res.get('image') item = get_metadata_newspaper(item) # yield item if item: items.append(item) return iter(items) except Exception as error: logger.error(error)
def get_recommendation(category, filter_type=None, filter_value=None): """ Get recommended articles :param category: Category from RSS type :param filter_type: Filter type :param filter_value: Filter value :return: RecommendedItems """ try: if filter_type == 'tags' and filter_value and not category: query = {"summary": {"$regex": filter_value, "$options": 'i'}} results = list( db.items.find(query, { "_id": 0, "title": 1, "summary": 1, "link_url": 1 })) return {"Similar Articles": results, "Chosen Tag": filter_value} elif filter_type == 'tags' and filter_value and category: query = { "summary": { "$regex": filter_value, "$options": "i" }, "rss_categories": category } results = list( db.items.find(query, { "_id": 0, "title": 1, "summary": 1, "link_url": 1 })) return {"Similar Articles": results, "Chosen Tag": filter_value} # # elif filter_type == 'source' and filter_value: # results = list(db.items.find({"rss_categories": category, "provider": "provider"}, # {"_id": 0, "title": 1, "summary": 1, "link_url": 1})) # return {"Similar Articles": results, "Chosen Provider": provider, "Chosen Category": category} else: results = list( db.items.find({"rss_categories": category}, { "_id": 0, "title": 1, "summary": 1, "link_url": 1 })) return {"Similar Articles": results, "Chosen Category": category} return results except Exception as error: logger.error(error) #if __name__ == '__main__': #pass
def execute_cmd(self, cmd: str): """ :param cmd: 服务器下对应的命令, 可以是list,或者str """ stdin, stdout, stderr = self.ssh.exec_command(cmd) error = stderr.read().decode() logger.info(f"输入命令: {cmd} -> 输出结果: {stdout.read().decode()}") logger.error(f"异常信息: {error}") return error
def recommendation(): try: category = request.args.get('categories') filter_type = request.args.get('filter_type') filter_value = request.args.get('filter_value') results = get_recommendation(category, filter_type, filter_value) return jsonify(results) except Exception as error: logger.error(error)
def generateFabricCerts(yamlContent): clusterName = yamlContent["clusterName"] mountPoint = yamlContent["nfsServer"]["mountPoint"] clusterPath = mountPoint + "/" + clusterName resourcesPath = clusterPath + "/resources" command = "env CONFIG_PATH=" + resourcesPath + " FABRIC_CFG_PATH=" + resourcesPath + " cryptogen generate \ --config=" + resourcesPath + "/crypto-config.yaml --output=" + resourcesPath + "/crypto-config" re = os.system(command) if re != 0: logger.error(command + " exec failed.") sys.exit(1)
def insert(**kwargs): ''' Expects DB fields as kwargs Flush changes to session ''' new_record = FLT(**kwargs) try: session.add(new_record) session.flush() except OperationalError: logger.error('Insert new person failed') raise MyLocalException
def build(self, file_path): with open(file_path) as fin: for line_no, line in enumerate(fin): tmp_list = line.split('\t') if not tmp_list or len(tmp_list) != 2: logger.error("split line[%s] error" %line) continue title = tmp_list[0].strip() link = tmp_list[1].strip() segment_list = self._segment(title) if segment_list : self._build(file_path, line_no, segment_list)
def f_retry(*args, **kwargs): """retry function""" mtries, mdelay = tries, delay # make mutable ret = func(*args, **kwargs) # first attempt while mtries > 0: if ret != None or type(ret) == str or type(ret) == dict: return ret mtries -= 1 # consume an attempt time.sleep(mdelay) # wait... mdelay *= backoff # make future wait longer ret = func(*args, **kwargs) # Try again logger.error("couldn't connect to report server.") return False # Ran out of tries
def _check_input(cls, data): # should not be overwritten, prefer _check_data if isinstance(data, cls): return data._data elif isinstance(data, str) or data is None: return data elif isiterable(data) and cls._ALLOW_MULTIPLE or cls._ALLOW_CASTING: return data elif cls._SILENT_ERROR: return None else: msg = "argument should be a path-like object or str, not '{}'".format(type(data)) logger.error(msg) raise TypeError(msg)
def remove(self, mode, plant): if mode not in ['remove', 'activate', 'deactivate']: raise ValueError('remove mode not valid') if not plant.localhost: daemon = MeshNetwork() # Plant.active == True, for target in list(Plant.select().where(Plant.localhost == False, Plant.role == 'master')): initial = {'mode': mode, 'destination': {'uuid': str(plant.uuid), 'ip': plant.ip, 'relation': 'master'}} daemon.remove(1, 1, target, initial=initial) status = self.get(120) if status == 1: logger.info('successful') else: logger.info('could not reach host') local = Plant.get(localhost=True) if plant.role != 'master' and plant.role == str(local.uuid): daemon.remove(2, 1, plant) status = self.get(120) if status == 1: logger.info('successful') else: logger.info('could not reach host') if mode == 'remove': local = Plant.get(localhost=True) for slave in list(Plant.select().where(Plant.role == str(plant.uuid))): slave.role = str(local.uuid) slave.save() from models.sensor import SensorData, SensorStatus, SensorCount, SensorSatisfactionValue, SensorDataPrediction from models.plant import PlantNetworkUptime for model in [PlantNetworkUptime, SensorData, SensorStatus, SensorCount, SensorSatisfactionValue, SensorDataPrediction]: model.delete().where(model.plant == plant).execute() plant.delete_instance() elif mode == 'activate': plant.active = True plant.save() elif mode == 'deactivate': plant.active = False plant.save() else: logger.error('not supported mode') return True else: return False
def _Preprocess(self, engine): logger.debug('Starting preprocessing.') try: artifacts_registry = process_engine.ProcessEngine.BuildArtifactsRegistry( self._artifact_definitions_path, self._custom_artifacts_path) engine.Preprocess(artifacts_registry, self._source_path_specs, resolver_context=self._resolver_context) except IOError as exception: logger.error( 'Unable to preprocess with error: {0!s}'.format(exception)) logger.debug('Preprocessing done.')
def make_json(self): ''' Expects abspath to .csv file Returns json data ''' flt = self.parse_filename() passengers = self.csv_to_dict() flt['prl'] = passengers try: json_data = json.dumps(flt) logger.info(f'{self.file_name}; Convert to json - OK') return json_data except (AttributeError, TypeError): logger.error(f'{self.file_name}; Convert to json - Error') raise MyLocalException
def checkIfClusterExists(yamlContent): clusterName = yamlContent["clusterName"] mountPoint = yamlContent["nfsServer"]["mountPoint"] clusterPath = mountPoint + "/" + clusterName lockfile = clusterPath + "/" + "install.lock" if not os.path.exists(clusterPath): re = os.makedirs(clusterPath) if re != None: errMsg = "Create dir " + clusterPath + "failed." logger.error(errMsg) sys.exit(1) dstDir = clusterPath + "/" + "resources" shutil.copytree("resources", dstDir) return False else: return os.path.exists(lockfile)
def fetch(): json_data_client = client_info.CLIENTS if not json_data_client: logger.warning( "RSS client info is empty. Check the client_info.py file") sys.exit(1) try: # Loop through the clients and individual category RSS site link for client in json_data_client: base_url = json_data_client[client]["base_url"] for category in json_data_client[client]["endpoints"]: url = base_url + json_data_client[client]["endpoints"][category] pull_feeds(url, client, category) except Exception as error: logger.error(error)
def update_metadata(item, rss_cat, provider): """ Update the metadata of the content and structure the data to be pushed into DB :param item: Individual item :param rss_cat: Category of the article :param provider: Provider :return: result """ try: title = item.get('title', '') description = item.get('description', '') #@TODO: Use compile description = re.sub(spl_chr_pattern, '', description) tags = [] topics = [] classifications = [] if item.get('published'): publish_date = datetime.now() #publish_date = item.get('published').strftime('%d-%m-%Y') else: publish_date = datetime.now() # Update the dicitonary RSS provider data_point result = {} result['item_id'] = str( uuid.uuid4()) #hashlib.md5(title.encode('utf-8')).hexdigest() result['provider'] = provider result['language'] = "English" result['title'] = title result['description'] = description result['summary'] = item.get('summary') result['tags'] = tags result['topics'] = topics result['rss_categories'] = rss_cat result['image_url'] = item.get('image_url', '') result['link_url'] = item.get('link', '') result['classifications'] = classifications result['publish_date'] = publish_date result['last_modified'] = datetime.now() return result except Exception as error: logger.error(error)
def ScanFileObject(self, file_object): if not file_object: return try: scan_state = pysigscan.scan_state() self._scanner.scan_file_object(scan_state, file_object) """scan_state.set_data_size(len(file_content)) self._scanner.scan_start(scan_state) self._scanner.scan_buffer(scan_state, file_content) self._scanner.scan_stop(scan_state)""" except IOError as exception: logger.error('unable to scan file: error') return False #return scan_state.number_of_scan_results > 0 return scan_state.scan_results
def parse_filename(self) -> dict: ''' Expects abspath to .csv file Parse .csv filename and put result to dict Returns python dict ''' try: fields = self.file_name.replace('.csv', '').split('_') date = fields[0] date = dt.strptime(date, '%Y%m%d') date = dt.strftime(date, '%Y-%m-%d') flt = int(fields[1]) dep = fields[2] summary = {'flt': flt, 'date': date, 'dep': dep} return summary except (IndexError, ValueError, TypeError): logger.error(f'{self.file_name}; Incorrect filename') raise MyLocalException
def login(): """ return access_token """ global access_token global expires_date if expires_date > int(time.time()) and len(access_token) > 0: return access_token url = "https://openapi.baidu.com/oauth/2.0/token?grant_type=client_credentials&client_id=6NCOZQHM7f2bGzc9tKemZovU&client_secret=X0uXNGMIUkiockwY8Q16P6B41E3Xc98a&" resStr = requests.get(url) res = resStr.json() try: access_token = res["access_token"] expires_date = int(time.time()) + res["expires_in"] - 1000 return access_token except: import traceback traceback.print_exc() error("获取百度 tts token 失败:" + resStr.text()) access_token = "" expires_date = 0 return ""
def run(self): if not os.path.isfile(pid_file) and VariousTools.verify_database(): with open(pid_file, 'w') as output: output.write(str(os.getpid())) try: while True: sleep_seconds = self.next_execution_seconds() time.sleep(sleep_seconds) if not DUMMYPLANT: exc = Process(target=self.execute) exc.daemon = True exc.start() logger.debug('mode: real') else: exc = Process(target=self.simulate) exc.daemon = True exc.start() logger.debug('mode: simulated') except KeyboardInterrupt: print('Bye!') sys.exit() except Exception as e: if DEBUG is False: self.run() else: print(e) finally: self.exit() else: if not VariousTools.verify_database(): logger.error( 'aborted action - database required - no database provided' ) if os.path.isfile(pid_file): logger.error('process already running')
def save_to_db(json_data): ''' Parse json to DB fields Put them to session with insert() Commit changes ''' data = json.loads(json_data) flt = data['flt'] depdate = data['date'] depdate = dt.strptime(depdate, '%Y-%m-%d') dep = data['dep'] try: for person in data['prl']: insert(name=f"{person['surname']} {person['firstname']}", flt=flt, depdate=depdate, dep=dep) session.commit() logger.info('Commit to DB - OK') except OperationalError: logger.error('Commit to DB - Error') raise MyLocalException
def checkConsensusServiceStatus(yamlContent): logger.info("Checking consensus service status.") clusterName = yamlContent["clusterName"] ordererOrgName=yamlContent["crypto-config.yaml"]["OrdererOrgs"][0]["Name"] ORDERMSPID="OrdererMSP" ordererDomain = yamlContent["crypto-config.yaml"]["OrdererOrgs"][0]["Domain"] MSPCONFIGPATH = "/etc/hyperledger/ordererOrganizations/" + ordererDomain + "/orderers/orderer0." + ordererDomain + "/msp" ordererNamespace=ordererOrgName.lower() + "-" + clusterName.lower() peerOrgs = yamlContent["crypto-config.yaml"]["PeerOrgs"] checkCommand="env CORE_PEER_LOCALMSPID=" + ORDERMSPID + " " + "CORE_PEER_MSPCONFIGPATH=" + MSPCONFIGPATH + " " + \ "peer channel fetch 0 -o " + "orderer0." + ordererNamespace + ":7050 -c testchainid" re = 0 timeCount = 0 config.load_kube_config() v1 = client.CoreV1Api() ret = v1.list_pod_for_all_namespaces(watch=False) while True: for peerOrg in peerOrgs: peerOrgName=peerOrg["Name"] namespace=peerOrgName.lower() + "-" + clusterName.lower() for i in ret.items: if (i.metadata.namespace == namespace and i.metadata.name.startswith("cli")): cliPodName = i.metadata.name resp = execCmdInPod(cliPodName, namespace, checkCommand) logger.debug(resp) re = resp.find("Error") if re == -1: break else: timeCount = timeCount + INTERVAL time.sleep(INTERVAL) if timeCount > TIMEOUT: errMsg="Error: Consensus service is not healthy" logger.error(errMsg) sys.exit(1) logger.info("Consensus service is OK.")
def Write(self, string): """Writes a string to the output. Args: string (str): output. """ try: # Note that encode() will first convert string into a Unicode string # if necessary. encoded_string = codecs.encode(string, self._encoding, self._errors) except UnicodeEncodeError: if self._errors == 'strict': logger.error( 'Unable to properly write output due to encoding error. ' 'Switching to error tolerant encoding which can result in ' 'non Basic Latin (C0) characters to be replaced with "?" or ' '"\\ufffd".') self._errors = 'replace' encoded_string = codecs.encode(string, self._encoding, self._errors) self._file_object.write(encoded_string)
def generateChannelArtifacts(yamlContent): clusterName = yamlContent["clusterName"] mountPoint = yamlContent["nfsServer"]["mountPoint"] peerOrgs = yamlContent["crypto-config.yaml"]["PeerOrgs"] channelName = yamlContent["channelName"] clusterPath = mountPoint + "/" + clusterName resourcesPath = clusterPath + "/resources" channelArtifacts = resourcesPath + "/channel-artifacts" re = os.makedirs(channelArtifacts, exist_ok=True) if re != None: errMsg = "Create " + channelArtifacts + " Dir failed." logger.error(errMsg) sys.exit(1) command = "env CONFIG_PATH=" + resourcesPath + " FABRIC_CFG_PATH=" + resourcesPath + " configtxgen \ -profile OrdererGenesis -outputBlock " + channelArtifacts + "/genesis.block" re = os.system(command) if re != 0: logger.error(command + " exec failed.") sys.exit(1) #configtxgen -profile DefaultChannel -outputCreateChannelTx ./channel-artifacts/channel.tx -channelID $CHANNEL_NAME command = "env CONFIG_PATH=" + resourcesPath + " FABRIC_CFG_PATH=" + resourcesPath + " configtxgen \ -profile DefaultChannel -outputCreateChannelTx " + channelArtifacts + "/channel.tx -channelID " + channelName re = os.system(command) if re != 0: logger.error(command + " exec failed.") sys.exit(1) for peerOrg in peerOrgs: peerOrgName = peerOrg["Name"] command = "env CONFIG_PATH=" + resourcesPath + " FABRIC_CFG_PATH=" + resourcesPath + " configtxgen -profile \ DefaultChannel -outputAnchorPeersUpdate " + channelArtifacts + "/" + peerOrgName + "MSPanchors.tx -channelID " \ + channelName + " -asOrg " + peerOrgName + "MSP" re = os.system(command) if re != 0: logger.error(command + " exec failed.") sys.exit(1)
def app_url(): apps_url = [] directory = os.path.join(BASE_DIR, "apps") try: for filename in os.listdir(directory): try: full_path = os.path.join(directory, filename) if os.path.isdir(full_path) and os.path.exists(os.path.join(full_path, "urls.py")): apps_url += getattr( __import__(os.path.join("apps", filename, "urls").replace("/", "."), fromlist=["url"]), "url" ) except Exception as e: print e, ":url", filename logger.error(full_path) logger.error(e) except Exception as e: print e, ":url error" logger.error(e) return apps_url
def parse(self, file_path): xml_dicts = None try: xml_dicts = self.__parser.parse(file_path) except Exception,e: logger.error("%s file parse failed. [%s]" %(file_path, e))