def ListTimeZones(self):
"""Lists the timezones."""
max_length = 0
for timezone_name in pytz.all_timezones:
if len(timezone_name) > max_length:
max_length = len(timezone_name)
utc_date_time = datetime.datetime.utcnow()
print('{0:20}\t{1:s}'.format('TimezoneUTC', 'Offset'))
for timezone_name in pytz.all_timezones:
try:
local_timezone = pytz.timezone(timezone_name)
except AssertionError as exception:
logger.error((
'Unable to determine information about timezone: {0:s} with '
'error: {1!s}').format(timezone_name, exception))
continue
local_date_string = '{0!s}'.format(
local_timezone.localize(utc_date_time))
if '+' in local_date_string:
_, _, diff = local_date_string.rpartition('+')
diff_string = '+{0:s}'.format(diff)
else:
_, _, diff = local_date_string.rpartition('-')
diff_string = '-{0:s}'.format(diff)
print('{0:20}\t{1:s}'.format(timezone_name, diff_string))
def pull_feeds(url, provider, rss_category):
"""
Pull feeds for an individual category specific Article
:param url: Category based Provider URL link
:param provider: Name of the Article Provider
:param rss_category: Category of the articles
:return: None
"""
try:
# Get provider from the config file
items = parse_url(url)
if items:
# Loop through the items iterator object
for item in items:
# if no title or description skip the article
if not item['title'] and item['description']:
continue
# Update Metadata
result = update_metadata(item, rss_category, provider)
# Update Mongo DB
update_mongo(result, provider)
else:
logger.warning(
"Unable to pull items for `{}` in `{}` category".format(
provider, rss_category))
except Exception as error:
logger.error(error)
def checkFabricServiceStatus(yamlContent):
logger.info("Waiting for BoK to start up ...")
clusterName = yamlContent["clusterName"]
peerOrgs = yamlContent["crypto-config.yaml"]["PeerOrgs"]
ifReady=0
timeCount=0
config.load_kube_config()
v1 = client.CoreV1Api()
while True:
ret = v1.list_pod_for_all_namespaces(watch=False)
for peerOrg in peerOrgs:
peerOrgName=peerOrg["Name"]
namespace=peerOrgName.lower() + "-" + clusterName.lower()
for i in ret.items:
if namespace == i.metadata.namespace:
if i.status.phase != "Running":
ifReady=0
break
else:
ifReady=1
if ifReady == 0:
break
if ifReady == 1:
logger.info("BoK is up and running.")
break
else:
timeCount = timeCount + INTERVAL
time.sleep(INTERVAL)
if timeCount > TIMEOUT:
logger.error("Error: Failed to start BoK service.")
sys.exit(1)
def get_metadata_newspaper(item):
"""
Use newspaper3k library to get more details like Image URL and other metadata
:param item: Article data
:return: item
"""
try:
if not item['link'] and not item['id']:
return {}
if item['link'] and item['link'] != "":
article = Article(item['link'])
else:
article = Article(item['id'])
try:
article.download()
article.parse()
except ArticleException:
try:
article = Article(item['link'])
article.download()
article.parse()
except:
article = {}
if article:
if article.title:
item['title'] = article.title
if article.text:
item['description'] = article.text
if article.top_image:
item['image_url'] = article.top_image
except Exception as error:
logger.error(error)
return item
def checkAndRun(f):
if os.path.isfile(f):
# sleep 5 seconds before deployment creation,
# in case the PV hasn't initialized completely.
os.system("kubectl apply -f " + f)
else:
logger.error("The file %s does not exist." % (f))
def CreateStorageAndTempPath(self, cursor, case_id=None, evd_id=None):
if not case_id or not evd_id:
print("ERROR: Missing Case ID or Evidence ID.")
return
if self._source_path is None:
query = 'SELECT evd_path FROM evidence_info WHERE case_id="' + case_id + '" AND evd_id="' + evd_id + '";'
try:
evd_path = cursor.execute_query(query)
if evd_path is None:
raise ValueError
self._source_path = os.path.join(self._root_storage_path,
evd_path[0])
except ValueError:
logger.error('Evidence is not exist: \'evd_path\' is None')
raise Exception('Evidence is not exist: \'evd_path\' is None')
except Exception as exception:
logger.error(
'database execution failed: {0!s}'.format(exception))
raise Exception(
'database execution failed: {0!s}'.format(exception))
if not os.path.isdir(os.path.join(self._root_tmp_path, case_id)):
os.mkdir(os.path.join(self._root_tmp_path, case_id))
self._tmp_path = os.path.join(
os.path.join(self._root_tmp_path, case_id), evd_id)
if not os.path.isdir(self._tmp_path):
os.mkdir(self._tmp_path)
def pull_feeds():
try:
pull_rss_feedparser.fetch()
return jsonify({"Finished pulling all data!!!"})
except Exception as error:
logger.error(error)
def Read(self):
"""Reads a string from the input.
Returns:
str: input.
"""
encoded_string = self._file_object.readline()
if isinstance(encoded_string, str):
return encoded_string
try:
string = codecs.decode(encoded_string, self._encoding,
self._errors)
except UnicodeDecodeError:
if self._errors == 'strict':
logger.error(
'Unable to properly read input due to encoding error. '
'Switching to error tolerant encoding which can result in '
'non Basic Latin (C0) characters to be replaced with "?" or '
'"\\ufffd".')
self._errors = 'replace'
string = codecs.decode(encoded_string, self._encoding,
self._errors)
return string
def parse_url(url):
"""
Parse RSS XML URL and get the basic details
:param url: RSS XML URL
:param provider: Name of the Article provider
:return: item
"""
try:
items = []
column_list = [
'title', 'summary', 'id', 'language', 'link', 'description',
'published', 'media_content', 'image'
]
elements = feedparser.parse(url)
elements = (element for element in elements.entries)
if not elements:
return []
logger.info("Please wait, parsing through the RSS feeds........")
for element_res in elements:
item = {}
for element in element_res:
if element in column_list:
item[element] = element_res.get(element)
if 'media_content' in element_res:
item['image_url'] = element_res.media_content[0]['url']
else:
item['image_url'] = element_res.get('image')
item = get_metadata_newspaper(item)
# yield item
if item:
items.append(item)
return iter(items)
except Exception as error:
logger.error(error)
def get_recommendation(category, filter_type=None, filter_value=None):
"""
Get recommended articles
:param category: Category from RSS type
:param filter_type: Filter type
:param filter_value: Filter value
:return: RecommendedItems
"""
try:
if filter_type == 'tags' and filter_value and not category:
query = {"summary": {"$regex": filter_value, "$options": 'i'}}
results = list(
db.items.find(query, {
"_id": 0,
"title": 1,
"summary": 1,
"link_url": 1
}))
return {"Similar Articles": results, "Chosen Tag": filter_value}
elif filter_type == 'tags' and filter_value and category:
query = {
"summary": {
"$regex": filter_value,
"$options": "i"
},
"rss_categories": category
}
results = list(
db.items.find(query, {
"_id": 0,
"title": 1,
"summary": 1,
"link_url": 1
}))
return {"Similar Articles": results, "Chosen Tag": filter_value}
#
# elif filter_type == 'source' and filter_value:
# results = list(db.items.find({"rss_categories": category, "provider": "provider"},
# {"_id": 0, "title": 1, "summary": 1, "link_url": 1}))
# return {"Similar Articles": results, "Chosen Provider": provider, "Chosen Category": category}
else:
results = list(
db.items.find({"rss_categories": category}, {
"_id": 0,
"title": 1,
"summary": 1,
"link_url": 1
}))
return {"Similar Articles": results, "Chosen Category": category}
return results
except Exception as error:
logger.error(error)
#if __name__ == '__main__':
#pass
def execute_cmd(self, cmd: str):
"""
:param cmd: 服务器下对应的命令, 可以是list，或者str
"""
stdin, stdout, stderr = self.ssh.exec_command(cmd)
error = stderr.read().decode()
logger.info(f"输入命令: {cmd} -> 输出结果: {stdout.read().decode()}")
logger.error(f"异常信息: {error}")
return error
def recommendation():
try:
category = request.args.get('categories')
filter_type = request.args.get('filter_type')
filter_value = request.args.get('filter_value')
results = get_recommendation(category, filter_type, filter_value)
return jsonify(results)
except Exception as error:
logger.error(error)
def generateFabricCerts(yamlContent):
clusterName = yamlContent["clusterName"]
mountPoint = yamlContent["nfsServer"]["mountPoint"]
clusterPath = mountPoint + "/" + clusterName
resourcesPath = clusterPath + "/resources"
command = "env CONFIG_PATH=" + resourcesPath + " FABRIC_CFG_PATH=" + resourcesPath + " cryptogen generate \
--config=" + resourcesPath + "/crypto-config.yaml --output=" + resourcesPath + "/crypto-config"
re = os.system(command)
if re != 0:
logger.error(command + " exec failed.")
sys.exit(1)
def insert(**kwargs):
'''
Expects DB fields as kwargs
Flush changes to session
'''
new_record = FLT(**kwargs)
try:
session.add(new_record)
session.flush()
except OperationalError:
logger.error('Insert new person failed')
raise MyLocalException
def build(self, file_path):
with open(file_path) as fin:
for line_no, line in enumerate(fin):
tmp_list = line.split('\t')
if not tmp_list or len(tmp_list) != 2:
logger.error("split line[%s] error" %line)
continue
title = tmp_list[0].strip()
link = tmp_list[1].strip()
segment_list = self._segment(title)
if segment_list :
self._build(file_path, line_no, segment_list)
def f_retry(*args, **kwargs):
"""retry function"""
mtries, mdelay = tries, delay # make mutable
ret = func(*args, **kwargs) # first attempt
while mtries > 0:
if ret != None or type(ret) == str or type(ret) == dict:
return ret
mtries -= 1 # consume an attempt
time.sleep(mdelay) # wait...
mdelay *= backoff # make future wait longer
ret = func(*args, **kwargs) # Try again
logger.error("couldn't connect to report server.")
return False # Ran out of tries
def _check_input(cls, data): # should not be overwritten, prefer _check_data
if isinstance(data, cls):
return data._data
elif isinstance(data, str) or data is None:
return data
elif isiterable(data) and cls._ALLOW_MULTIPLE or cls._ALLOW_CASTING:
return data
elif cls._SILENT_ERROR:
return None
else:
msg = "argument should be a path-like object or str, not '{}'".format(type(data))
logger.error(msg)
raise TypeError(msg)
def remove(self, mode, plant):
if mode not in ['remove', 'activate', 'deactivate']:
raise ValueError('remove mode not valid')
if not plant.localhost:
daemon = MeshNetwork()
# Plant.active == True,
for target in list(Plant.select().where(Plant.localhost == False, Plant.role == 'master')):
initial = {'mode': mode, 'destination': {'uuid': str(plant.uuid), 'ip': plant.ip, 'relation': 'master'}}
daemon.remove(1, 1, target, initial=initial)
status = self.get(120)
if status == 1:
logger.info('successful')
else:
logger.info('could not reach host')
local = Plant.get(localhost=True)
if plant.role != 'master' and plant.role == str(local.uuid):
daemon.remove(2, 1, plant)
status = self.get(120)
if status == 1:
logger.info('successful')
else:
logger.info('could not reach host')
if mode == 'remove':
local = Plant.get(localhost=True)
for slave in list(Plant.select().where(Plant.role == str(plant.uuid))):
slave.role = str(local.uuid)
slave.save()
from models.sensor import SensorData, SensorStatus, SensorCount, SensorSatisfactionValue, SensorDataPrediction
from models.plant import PlantNetworkUptime
for model in [PlantNetworkUptime, SensorData, SensorStatus, SensorCount, SensorSatisfactionValue, SensorDataPrediction]:
model.delete().where(model.plant == plant).execute()
plant.delete_instance()
elif mode == 'activate':
plant.active = True
plant.save()
elif mode == 'deactivate':
plant.active = False
plant.save()
else:
logger.error('not supported mode')
return True
else:
return False
def _Preprocess(self, engine):
logger.debug('Starting preprocessing.')
try:
artifacts_registry = process_engine.ProcessEngine.BuildArtifactsRegistry(
self._artifact_definitions_path, self._custom_artifacts_path)
engine.Preprocess(artifacts_registry,
self._source_path_specs,
resolver_context=self._resolver_context)
except IOError as exception:
logger.error(
'Unable to preprocess with error: {0!s}'.format(exception))
logger.debug('Preprocessing done.')
def make_json(self):
'''
Expects abspath to .csv file
Returns json data
'''
flt = self.parse_filename()
passengers = self.csv_to_dict()
flt['prl'] = passengers
try:
json_data = json.dumps(flt)
logger.info(f'{self.file_name}; Convert to json - OK')
return json_data
except (AttributeError, TypeError):
logger.error(f'{self.file_name}; Convert to json - Error')
raise MyLocalException
def checkIfClusterExists(yamlContent):
clusterName = yamlContent["clusterName"]
mountPoint = yamlContent["nfsServer"]["mountPoint"]
clusterPath = mountPoint + "/" + clusterName
lockfile = clusterPath + "/" + "install.lock"
if not os.path.exists(clusterPath):
re = os.makedirs(clusterPath)
if re != None:
errMsg = "Create dir " + clusterPath + "failed."
logger.error(errMsg)
sys.exit(1)
dstDir = clusterPath + "/" + "resources"
shutil.copytree("resources", dstDir)
return False
else:
return os.path.exists(lockfile)
def fetch():
json_data_client = client_info.CLIENTS
if not json_data_client:
logger.warning(
"RSS client info is empty. Check the client_info.py file")
sys.exit(1)
try:
# Loop through the clients and individual category RSS site link
for client in json_data_client:
base_url = json_data_client[client]["base_url"]
for category in json_data_client[client]["endpoints"]:
url = base_url + json_data_client[client]["endpoints"][category]
pull_feeds(url, client, category)
except Exception as error:
logger.error(error)
def update_metadata(item, rss_cat, provider):
"""
Update the metadata of the content and structure the data to be pushed into DB
:param item: Individual item
:param rss_cat: Category of the article
:param provider: Provider
:return: result
"""
try:
title = item.get('title', '')
description = item.get('description', '')
#@TODO: Use compile
description = re.sub(spl_chr_pattern, '', description)
tags = []
topics = []
classifications = []
if item.get('published'):
publish_date = datetime.now()
#publish_date = item.get('published').strftime('%d-%m-%Y')
else:
publish_date = datetime.now()
# Update the dicitonary RSS provider data_point
result = {}
result['item_id'] = str(
uuid.uuid4()) #hashlib.md5(title.encode('utf-8')).hexdigest()
result['provider'] = provider
result['language'] = "English"
result['title'] = title
result['description'] = description
result['summary'] = item.get('summary')
result['tags'] = tags
result['topics'] = topics
result['rss_categories'] = rss_cat
result['image_url'] = item.get('image_url', '')
result['link_url'] = item.get('link', '')
result['classifications'] = classifications
result['publish_date'] = publish_date
result['last_modified'] = datetime.now()
return result
except Exception as error:
logger.error(error)
def ScanFileObject(self, file_object):
if not file_object:
return
try:
scan_state = pysigscan.scan_state()
self._scanner.scan_file_object(scan_state, file_object)
"""scan_state.set_data_size(len(file_content))
self._scanner.scan_start(scan_state)
self._scanner.scan_buffer(scan_state, file_content)
self._scanner.scan_stop(scan_state)"""
except IOError as exception:
logger.error('unable to scan file: error')
return False
#return scan_state.number_of_scan_results > 0
return scan_state.scan_results
def parse_filename(self) -> dict:
'''
Expects abspath to .csv file
Parse .csv filename and put result to dict
Returns python dict
'''
try:
fields = self.file_name.replace('.csv', '').split('_')
date = fields[0]
date = dt.strptime(date, '%Y%m%d')
date = dt.strftime(date, '%Y-%m-%d')
flt = int(fields[1])
dep = fields[2]
summary = {'flt': flt,
'date': date,
'dep': dep}
return summary
except (IndexError, ValueError, TypeError):
logger.error(f'{self.file_name}; Incorrect filename')
raise MyLocalException
def login():
"""
return access_token
"""
global access_token
global expires_date
if expires_date > int(time.time()) and len(access_token) > 0:
return access_token
url = "https://openapi.baidu.com/oauth/2.0/token?grant_type=client_credentials&client_id=6NCOZQHM7f2bGzc9tKemZovU&client_secret=X0uXNGMIUkiockwY8Q16P6B41E3Xc98a&"
resStr = requests.get(url)
res = resStr.json()
try:
access_token = res["access_token"]
expires_date = int(time.time()) + res["expires_in"] - 1000
return access_token
except:
import traceback
traceback.print_exc()
error("获取百度 tts token 失败:" + resStr.text())
access_token = ""
expires_date = 0
return ""
def run(self):
if not os.path.isfile(pid_file) and VariousTools.verify_database():
with open(pid_file, 'w') as output:
output.write(str(os.getpid()))
try:
while True:
sleep_seconds = self.next_execution_seconds()
time.sleep(sleep_seconds)
if not DUMMYPLANT:
exc = Process(target=self.execute)
exc.daemon = True
exc.start()
logger.debug('mode: real')
else:
exc = Process(target=self.simulate)
exc.daemon = True
exc.start()
logger.debug('mode: simulated')
except KeyboardInterrupt:
print('Bye!')
sys.exit()
except Exception as e:
if DEBUG is False:
self.run()
else:
print(e)
finally:
self.exit()
else:
if not VariousTools.verify_database():
logger.error(
'aborted action - database required - no database provided'
)
if os.path.isfile(pid_file):
logger.error('process already running')
def save_to_db(json_data):
'''
Parse json to DB fields
Put them to session with insert()
Commit changes
'''
data = json.loads(json_data)
flt = data['flt']
depdate = data['date']
depdate = dt.strptime(depdate, '%Y-%m-%d')
dep = data['dep']
try:
for person in data['prl']:
insert(name=f"{person['surname']} {person['firstname']}",
flt=flt,
depdate=depdate,
dep=dep)
session.commit()
logger.info('Commit to DB - OK')
except OperationalError:
logger.error('Commit to DB - Error')
raise MyLocalException
def checkConsensusServiceStatus(yamlContent):
logger.info("Checking consensus service status.")
clusterName = yamlContent["clusterName"]
ordererOrgName=yamlContent["crypto-config.yaml"]["OrdererOrgs"][0]["Name"]
ORDERMSPID="OrdererMSP"
ordererDomain = yamlContent["crypto-config.yaml"]["OrdererOrgs"][0]["Domain"]
MSPCONFIGPATH = "/etc/hyperledger/ordererOrganizations/" + ordererDomain + "/orderers/orderer0." + ordererDomain + "/msp"
ordererNamespace=ordererOrgName.lower() + "-" + clusterName.lower()
peerOrgs = yamlContent["crypto-config.yaml"]["PeerOrgs"]
checkCommand="env CORE_PEER_LOCALMSPID=" + ORDERMSPID + " " + "CORE_PEER_MSPCONFIGPATH=" + MSPCONFIGPATH + " " + \
"peer channel fetch 0 -o " + "orderer0." + ordererNamespace + ":7050 -c testchainid"
re = 0
timeCount = 0
config.load_kube_config()
v1 = client.CoreV1Api()
ret = v1.list_pod_for_all_namespaces(watch=False)
while True:
for peerOrg in peerOrgs:
peerOrgName=peerOrg["Name"]
namespace=peerOrgName.lower() + "-" + clusterName.lower()
for i in ret.items:
if (i.metadata.namespace == namespace and i.metadata.name.startswith("cli")):
cliPodName = i.metadata.name
resp = execCmdInPod(cliPodName, namespace, checkCommand)
logger.debug(resp)
re = resp.find("Error")
if re == -1:
break
else:
timeCount = timeCount + INTERVAL
time.sleep(INTERVAL)
if timeCount > TIMEOUT:
errMsg="Error: Consensus service is not healthy"
logger.error(errMsg)
sys.exit(1)
logger.info("Consensus service is OK.")
def Write(self, string):
"""Writes a string to the output.
Args:
string (str): output.
"""
try:
# Note that encode() will first convert string into a Unicode string
# if necessary.
encoded_string = codecs.encode(string, self._encoding,
self._errors)
except UnicodeEncodeError:
if self._errors == 'strict':
logger.error(
'Unable to properly write output due to encoding error. '
'Switching to error tolerant encoding which can result in '
'non Basic Latin (C0) characters to be replaced with "?" or '
'"\\ufffd".')
self._errors = 'replace'
encoded_string = codecs.encode(string, self._encoding,
self._errors)
self._file_object.write(encoded_string)
def generateChannelArtifacts(yamlContent):
clusterName = yamlContent["clusterName"]
mountPoint = yamlContent["nfsServer"]["mountPoint"]
peerOrgs = yamlContent["crypto-config.yaml"]["PeerOrgs"]
channelName = yamlContent["channelName"]
clusterPath = mountPoint + "/" + clusterName
resourcesPath = clusterPath + "/resources"
channelArtifacts = resourcesPath + "/channel-artifacts"
re = os.makedirs(channelArtifacts, exist_ok=True)
if re != None:
errMsg = "Create " + channelArtifacts + " Dir failed."
logger.error(errMsg)
sys.exit(1)
command = "env CONFIG_PATH=" + resourcesPath + " FABRIC_CFG_PATH=" + resourcesPath + " configtxgen \
-profile OrdererGenesis -outputBlock " + channelArtifacts + "/genesis.block"
re = os.system(command)
if re != 0:
logger.error(command + " exec failed.")
sys.exit(1)
#configtxgen -profile DefaultChannel -outputCreateChannelTx ./channel-artifacts/channel.tx -channelID $CHANNEL_NAME
command = "env CONFIG_PATH=" + resourcesPath + " FABRIC_CFG_PATH=" + resourcesPath + " configtxgen \
-profile DefaultChannel -outputCreateChannelTx " + channelArtifacts + "/channel.tx -channelID " + channelName
re = os.system(command)
if re != 0:
logger.error(command + " exec failed.")
sys.exit(1)
for peerOrg in peerOrgs:
peerOrgName = peerOrg["Name"]
command = "env CONFIG_PATH=" + resourcesPath + " FABRIC_CFG_PATH=" + resourcesPath + " configtxgen -profile \
DefaultChannel -outputAnchorPeersUpdate " + channelArtifacts + "/" + peerOrgName + "MSPanchors.tx -channelID " \
+ channelName + " -asOrg " + peerOrgName + "MSP"
re = os.system(command)
if re != 0:
logger.error(command + " exec failed.")
sys.exit(1)
def app_url():
apps_url = []
directory = os.path.join(BASE_DIR, "apps")
try:
for filename in os.listdir(directory):
try:
full_path = os.path.join(directory, filename)
if os.path.isdir(full_path) and os.path.exists(os.path.join(full_path, "urls.py")):
apps_url += getattr(
__import__(os.path.join("apps", filename, "urls").replace("/", "."), fromlist=["url"]), "url"
)
except Exception as e:
print e, ":url", filename
logger.error(full_path)
logger.error(e)
except Exception as e:
print e, ":url error"
logger.error(e)
return apps_url
def parse(self, file_path):
xml_dicts = None
try:
xml_dicts = self.__parser.parse(file_path)
except Exception,e:
logger.error("%s file parse failed. [%s]" %(file_path, e))
