def importprojectdir(dir_project, file_type):
files = []
for file_name in glob.glob(os.path.join(dir_project, '*.' + file_type)):
files.append(Util().openfile(file_name))
for file_name in glob.glob(os.path.join(dir_project, '*.' + 'json')):
files.append(Util().openfile(file_name))
return importprojectfile(files)
def load(self):
print('load 1')
filelist = Util().getFileList(self.folder,".dep")
for fn in filelist:
#print('fn',fn, self.getOriginalName(fn))
if Util().file_exists(self.folder, self.getOriginalName(fn)):
print('found skip ', fn)
else:
#print('not found remove from git')
self.append(fn)
return self
def get_add_element(self, request):
result = False
group_id = request.POST.get('group_id')
element_id = request.POST.get('element_id')
element_type = request.POST.get('element_type')
current_data = json.loads(self.data_project)
tosca_nfv_definition = Util().loadyamlfile(PATH_TO_TOSCA_NFV_DEFINITION)
node_types = {}
node_types.update(tosca_nfv_definition['node_types'])
new_element = {}
new_element['type'] = element_type
type_definition = node_types[element_type]
while element_type in node_types:
type_definition = node_types[element_type]
if 'properties' in type_definition:
for propriety in type_definition['properties']:
if 'required' not in type_definition['properties'][propriety] or \
type_definition['properties'][propriety]['required']:
if 'properties' not in new_element:
new_element['properties'] = {}
if propriety == 'version':
new_element['properties'][propriety] = 1.0
else:
new_element['properties'][propriety] = 'prova'
element_type = type_definition['derived_from'] if 'derived_from' in type_definition else None
if new_element['type'] == 'tosca.nodes.nfv.VNF':
if 'imports' not in current_data['toscayaml'][group_id] or current_data['toscayaml'][group_id][
'imports'] is None:
current_data['toscayaml'][group_id]['imports'] = []
current_data['toscayaml'][group_id]['imports'].append(element_id + '.yaml')
vnf_template = Util().loadyamlfile(PATH_TO_DESCRIPTORS_TEMPLATES + 'vnf.yaml')
vnf_template['topology_template']['subsititution_mappings'] = 'tosca.nodes.nfv.VNF.' + element_id
vnf_template['topology_template']['node_templates'] = {}
vnf_template['imports'] = []
vnf_template['node_types']['tosca.nodes.nfv.VNF.' + element_id] = {}
vnf_template['node_types']['tosca.nodes.nfv.VNF.' + element_id]['derived_from'] = 'tosca.nodes.nfv.VNF'
current_data['toscayaml'][element_id] = vnf_template
if 'node_templates' not in current_data['toscayaml'][group_id]['topology_template'] or current_data['toscayaml'][group_id]['topology_template']['node_templates'] is None:
current_data['toscayaml'][group_id]['topology_template']['node_templates'] = {}
current_data['toscayaml'][group_id]['topology_template']['node_templates'][element_id] = new_element
self.data_project = current_data
# self.validated = validate #TODO(stefano) not clear if this is the validation for the whole project
self.update()
result = True
return result
def __requestAuthToken(self):
util = Util()
config = util.getPushConfig()
if config is None:
return None
requestTime = str(int(round(time.time() * 1000)))
sign = config['AppKey'] + requestTime + config['MasterSecret']
sh = hashlib.sha256()
sh.update(sign.encode("utf8"))
payload = {
"sign": sh.hexdigest(),
"timestamp": requestTime,
"appkey": config['AppKey']
}
try:
url = "https://restapi.getui.com/v1/" + config[
'AppId'] + "/auth_sign"
ret = util.httpsPost(url, params=payload)
if ret['result'] == 'ok':
self.__setAuthToken(ret)
return ret['auth_token']
except Exception as err:
print(err)
return None
def main():
opt = parse_cmdline()
flist = os.listdir(opt.yaml_dir)
ts = TreeShape(fanout=opt.fanout, width=900, height=900)
u = Util()
i = 0
for f in sorted(flist):
ybuf = u.file_to_yaml(f"{opt.yaml_dir}/{f}")
d = ts.process_yaml(ybuf)
if d != None:
ts.add_tree_data(d)
i += 1
p, slider = ts.plot_graph(title="cN Tree Shape")
if p == None:
print("Empty List")
return -1
if slider:
composite = [[p], [slider]]
else:
composite = [[p]]
output_file(opt.html_file)
plot = layout(composite)
save(plot)
print(f"Plot: {opt.html_file}")
def setUp(self):
super(Volume, self).setUp()
self.primary_node = self.input["node"]
self.util = Util(self.primary_node)
self.data_load_flag = False
self.number_of_cycles = 300 # ToDO: scale it up later
self.number_of_kv_pairs = 100000 # ToDO: scale it up later
def import_kubernetes_from_dir_project(cls, dir_project):
result = {}
for k8s_filename in glob.glob(
os.path.join(dir_project, 'K8S', '*.yaml')):
log.info(k8s_filename)
yaml_object = Util().loadyamlfile(k8s_filename)
json_object = Util.json_loads_byteified(
Util.yaml2json(yaml_object))
filename = os.path.splitext(os.path.basename(str(k8s_filename)))[0]
result[filename] = json_object
return result
def get_add_element(self, request):
result = False
group_id = request.POST.get('group_id')
element_id = request.POST.get('element_id')
element_type = request.POST.get('element_type')
x = request.POST.get('x')
y = request.POST.get('y')
current_data = json.loads(self.data_project)
tosca_definition = Util().loadyamlfile(PATH_TO_TOSCA_DEFINITION)
node_types = {}
node_types.update(tosca_definition['node_types'])
new_element = {}
new_element['type'] = element_type
type_definition = node_types[element_type]
while element_type in node_types:
type_definition = node_types[element_type]
if 'properties' in type_definition:
for propriety in type_definition['properties']:
if 'required' not in type_definition['properties'][
propriety] or type_definition['properties'][
propriety]['required']:
if 'properties' not in new_element:
new_element['properties'] = {}
if propriety == 'version':
new_element['properties'][propriety] = 1.0
else:
if type_definition['properties'][propriety][
'type'] == 'scalar-unit.size':
new_element['properties'][propriety] = '1 MB'
else:
new_element['properties'][propriety] = 'prova'
element_type = type_definition[
'derived_from'] if 'derived_from' in type_definition else None
if 'node_templates' not in current_data['toscayaml'][group_id][
'topology_template'] or current_data['toscayaml'][group_id][
'topology_template']['node_templates'] is None:
current_data['toscayaml'][group_id]['topology_template'][
'node_templates'] = {}
current_data['toscayaml'][group_id]['topology_template'][
'node_templates'][element_id] = new_element
if 'positions' not in current_data:
current_data['positions'] = {}
if 'vertices' not in current_data['positions']:
current_data['positions']['vertices'] = {}
if element_id not in current_data['positions']['vertices']:
current_data['positions']['vertices'][element_id] = {}
current_data['positions']['vertices'][element_id]['x'] = x
current_data['positions']['vertices'][element_id]['y'] = y
self.data_project = current_data
# self.validated = validate #TODO(stefano) not clear if this is the validation for the whole project
self.update()
result = True
return result
def get_descriptor_template(cls, type_descriptor):
"""Returns a descriptor template for a given descriptor type"""
try:
# schema = Util.loadjsonfile(PATH_TO_DESCRIPTORS_TEMPLATES+type_descriptor+DESCRIPTOR_TEMPLATE_SUFFIX)
# print 'type_descriptor : '+type_descriptor
# FixMe bisogna creare un template
yaml_object = Util().loadyamlfile(
'toscaparser/extensions/nfv/tests/data/tosca_helloworld_nfv.yaml')
toscajson = json.loads(Util.yaml2json(yaml_object))
return toscajson
except Exception as e:
# log.error('Exception in get descriptor template') #TODO(stefano) add logging
print 'Exception in get descriptor template'
return False
def __requestAccessToken(self):
util = Util()
config = util.getConfigByAgentId(self.agentId)
if config is None:
print('No configuration for '+ str(self.agentId) +' was found!')
return None
requestTime = str(int(time.time()))
try:
res = util.httpsGet('https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=' + config['CorpId'] + '&corpsecret=' + config['Secret'])
self.__setAccessToken({'expires_in': res['expires_in'], 'access_token': res['access_token'], 'request_time': requestTime})
return res['access_token']
except Exception as err:
print(err)
return None
def get_descriptor_template(cls, type_descriptor):
"""Returns a descriptor template for a given descriptor type"""
try:
#schema = Util.loadjsonfile(PATH_TO_DESCRIPTORS_TEMPLATES+type_descriptor+DESCRIPTOR_TEMPLATE_SUFFIX)
# print 'type_descriptor : '+type_descriptor
#FixMe bisogna creare un template
yaml_object = Util().loadyamlfile(
'usecases/TOSCA/One-Server-Three-Networks/YAML/tosca_one_server_three_networks.yaml'
)
toscajson = json.loads(Util.yaml2json(yaml_object))
return toscajson
except Exception as e:
# log.error('Exception in get descriptor template') #TODO(stefano) add logging
print 'Exception in get descriptor template'
return False
def importprojectfiles(cls, file_dict):
"""Imports descriptors (extracted from the new project POST)
The keys in the dictionary are the file types
"""
project = {'nsd': {}, 'vnfd': {}, 'click': {}, 'k8s': {}}
for desc_type in project:
if desc_type in file_dict:
files_desc_type = file_dict[desc_type]
for file in files_desc_type:
if desc_type != 'k8s':
project[desc_type][os.path.splitext(
file.name)[0]] = json.loads(file.read())
else:
yaml_object = Util().loadyamlfile(file)
json_object = Util.json_loads_byteified(
Util.yaml2json(yaml_object))
filename = os.path.splitext(os.path.basename(
str(file)))[0]
project[desc_type][filename] = json_object
return project
def make_plots(arg_dict):
u = Util()
vlen = arg_dict["vlen"]
nkey = arg_dict["nkey"]
tree_path = arg_dict["tree_path"]
crossover_path = arg_dict["crossover_path"]
title = "vlen: {} nkey: {} size: {}".format(vlen, u.humanize(nkey),
u.humanize(nkey * vlen))
heading = Div(
text=f"""
<hr style="width:100%">
<h2>{title}</h2>
""",
width=1000,
)
ts = TreeShape(fanout=16, width=500, height=500)
c = Crossover(width=1500, height=500)
ybuf = u.file_to_yaml(tree_path)
d = ts.process_yaml(ybuf)
ts.add_tree_data(d)
tot_kvsets, c_kvsets_avg, g_kvsets_avg = ts.avg_nkvset(ybuf, pivot=2)
tree_plot, slider = ts.plot_graph(
title="nkvsets Get: {} Cursor: {} Total: {}".format(
g_kvsets_avg, c_kvsets_avg, tot_kvsets))
df = c.df_for_nkeys(nkey_dir=crossover_path,
c_avg_nkvset=c_kvsets_avg,
g_avg_nkvset=g_kvsets_avg)
crossover_plot = c.plot(df=df)
# Output
arg_dict["plots"] = column(heading, row(tree_plot, crossover_plot))
def create_teams(self, client, message):
util = Util(client)
vc = util.GetAuthorVChannel(message)
if vc is None:
return "error:投稿者はボイスチャンネルに接続していません"
member_names = [a.display_name for a in vc.members]
member_names = self.modify_user_list(member_names, message)
randomized_names = random.sample(member_names, len(member_names))
team_num = self.set_team_num(message)
team_size = self.set_team_size(message)
if "-lol" in message.content:
if len(randomized_names) < 10:
return "error: 人数が10人未満です。"
teams = self.make_lol_team(randomized_names)
mes = self.lol_result(teams)
else:
teams = self.split_list(randomized_names, team_num)
mes = self.normal_result(teams)
return self.create_team_headder(team_num, team_size) + mes
def setUp(self):
super(SimpleTests, self).setUp()
self.primary_node = self.input["node"]
self.util = Util(self.primary_node)
db='one_db').setup()
print('gitDev', gitDev)
#system_folder='~/LyttleBit/projects/lb-_documents'
#in_folder = '../one_db/sql'
#out_folder = '../'
in_folder = gitDev.getDbFolder('sql')
#in_folder = '{}/{}'.format(system_folder,project_name)
#out_folder = '{}/{}'.format(system_folder,project_name)
out_folder = gitDev.getDbFolder()
#print('in_folder', in_folder)
#print('out_folder', out_folder)
outname = 'README.sql.md'
## READ: Make list of file in a folder
files = Util().getFileList(in_folder, ext='sql')
files.sort()
#print('files: ', files)
## PROCESS: Read all files
readme = []
#print('# Changes')
readme.append('\n')
readme.append('# Changes')
readme.append('\n')
readme.append('| file | type | detail |')
readme.append('| ---- | ------- | ------ |')
for f in files:
from random import randint
from threading import Timer
from lib.util import Util
from lib.wallet import Wallet
from lib.transaction import Transaction
utils = Util()
class Node():
def __init__(self, seed, *args):
self.seed = seed
self.wallet = Wallet(utils.hash(str(self.seed)))
self.neighbours = set()
self.DAG = {}
root_transaction = Transaction('0000', 'root', 25000000000)
self.attach_transaction(root_transaction, ['root'])
def proof_of_work(self, str_input):
proof = 0
while self.is_valid_proof(str_input, proof) is False:
proof += 1
return proof
def is_valid_proof(self, str_input, proof):
result = utils.hash(str_input + str(proof))
return result.count("00") >= 2 and result[-4:] == "0000"
def select_tips(self):
available_transactions = []
def main():
combined_sql_list = [] # eventually is written to a file
# [* Generate Heroku Scripts]
print('Generate Heroku Scripts')
print(' - Scripts to load, restart, and drop heroku applicatons')
# [* All configuration files are stored in /config folder]
config_folder = '{}'.format(os.getcwd().replace('_tasks', 'config'))
# [1. Select APP Source ]
sourceConfiguration = open_api(config_folder, file_type="source")
if not sourceConfiguration:
print('cancel')
exit(0)
# [2. Select APP Target ]
targetConfiguration = open_api(config_folder, file_type="target")
if not targetConfiguration:
print('cancel')
exit(0)
# [* Merge Source and Target]
sourceConfiguration.update(targetConfiguration)
apiConfiguration = sourceConfiguration
# setup default environment
homeDev = HomeDevelopment().setup()
# [* Create missing folders]
sourceDev = HomeDevelopment().setup()
targetDev = HomeDevelopment().setup()
# [* Scan configuration for home, source, and target environment configurations]
for apiName in apiConfiguration:
if apiName == 'source':
# [* Find Source configuration]
sourceDev = get_environment(apiConfiguration[apiName])
elif apiName == 'target':
# [* Find Target configuration]
targetDev = get_environment(apiConfiguration[apiName])
pprint(targetDev)
#exit(0)
print('=================')
report(homeDev, sourceDev, targetDev)
print('=================')
##########
# [## Heroku Scripts]
# [* generate a deployment script]
# [* generate a destroy script]
# [* generate a restart script]
#####
# configuration eg {"lb-a": {"heroku": {"operations":"C"}}}
apiNameList = [nm for nm in apiConfiguration
if 'heroku' in apiConfiguration[nm]
and 'operations' in apiConfiguration[nm]['heroku']
and 'C' in apiConfiguration[nm]['heroku']['operations']]
#apiNameList = [nm for nm in apiConfiguration
# if 'operations' in apiConfiguration[nm]['heroku']
# and 'C' in apiConfiguration[nm]['heroku']['operations'] ]
#apiNameList = [nm for nm in apiConfiguration if 'deploy' in apiConfiguration and apiConfiguration[nm]['deploy']]
print('apiNameList',apiNameList)
for appName in apiNameList:
print('=====',appName,'=====')
#pprint(apiConfiguration.flatten())
pprint(apiConfiguration.flatten(appName, parent_key='app'))
# [* Deployment script]
# [ * Dont overwrite scripts when they exist]
scriptFolder = targetDev.getFolder('sh-scripts')
# [* Create Heroku deployment script]
#scriptName = '{}.CREATE.sh'.format(appName)
scriptName = 'heroku.CREATE.sh'
overwrite = False
if Util().file_exists(scriptFolder, scriptName):
if Util().confirm('Overwrite {}'.format(scriptName), default='N') != 'N':
overwrite = True
if overwrite or not Util().file_exists(scriptFolder, scriptName):
#print('====Flatten=====')
tmpl = ScriptTemplate(appName,folder='../templates', filename='heroku.create.app.template') \
.apply(apiConfiguration.flatten()) \
.apply(apiConfiguration.flatten(appName, parent_key='app'))
tmpl.saveAs(scriptFolder, scriptName)
cmd = 'chmod 755 {}/{}'.format(scriptFolder, scriptName)
returned_value = subprocess.call(cmd, shell=True) # returns the exit code in unix
print('returned value:', returned_value)
overwrite = False
# [* Create Heroku destroy script]
#scriptName = '{}.DELETE.sh'.format(appName)
scriptName = 'heroku.DELETE.sh'
if Util().file_exists(scriptFolder, scriptName):
if Util().confirm('Overwrite {}'.format(scriptName), default='N') != 'N':
overwrite = True
if overwrite or not Util().file_exists(scriptFolder, scriptName):
#print('====Flatten=====')
tmpl = ScriptTemplate(appName,folder='../templates', filename='heroku.delete.app.template') \
.apply(apiConfiguration.flatten()) \
.apply(apiConfiguration.flatten(appName, parent_key='app'))
print('output to ', scriptFolder)
tmpl.saveAs(scriptFolder, scriptName)
cmd = 'chmod 755 {}/{}'.format(scriptFolder, scriptName)
returned_value = subprocess.call(cmd, shell=True) # returns the exit code in unix
print('returned value:', returned_value)
overwrite = False
import time
from bs4 import BeautifulSoup
from lib.dispatchUrl import DispatchUrl
from lib.util import Util
dispatchUrl = DispatchUrl()
caijiUtil = Util(host='http://www.cnedu.cn/')
beginUrls = [
'http://www.cnedu.cn/examination/courses/page1.shtm',
]
# 获取所有的的列表页
pageList = []
# 所有的url集合
urlList = []
index = 0
failLvl2Urls = open('./data/failLvl2Urls.txt', 'a')
failLvl1Urls = open('./data/failLvl1Urls.txt', 'a')
for beginUrl in beginUrls:
html = caijiUtil.getUrlContent(beginUrl)
if not html:
print("获取首页列表失败:: " + beginUrl)
failLvl1Urls.write(beginUrl)
continue
#soup = BeautifulSoup(html,"html5lib")
totoal = 1655
#获取所有列表页
#pageList.append(beginUrl)
for i in range(796, totoal):
url_ = beginUrl.replace('1.shtm', '%s.shtm' % (i))
pageList.append(url_)
#coding=utf-8
import time
from bs4 import BeautifulSoup
from lib.dispatchUrl import DispatchUrl
from lib.util import Util
dispatchUrl = DispatchUrl()
caijiUtil = Util(host='http://zhenti.kaoyan.eol.cn/')
beginUrls =[
'http://kaoyan.eol.cn/shiti/zhengzhi/index.shtml',
'http://kaoyan.eol.cn/shiti/yingyu/index.shtml',
'http://kaoyan.eol.cn/shiti/shuxue/index.shtml',
]
# 获取所有的的列表页
pageList = []
# 所有的url集合
urlList = []
index = 0
for beginUrl in beginUrls:
html = caijiUtil.getUrlContent(beginUrl)
if not html:
print("获取首页列表失败:: "+beginUrl)
continue
soup = BeautifulSoup(html,"html5lib")
pageList_ = soup.select('.page_left #pagenav')
totoal = caijiUtil.getTotalPage(str(pageList_[0]))
#获取所有列表页
pageList.append(beginUrl)
def main():
combined_sql_list = [] # eventually is written to a file
# [Generate API sql file]
print('Generate API')
print(' - load api configuration, generate funcPattern key and values')
# get configuration file name {folder: "", name: ""}
# get list of files of type .json in folder ./config
# [Use a configuration file]
config_folder = '{}'.format(os.getcwd().replace('_tasks', 'config'))
# [Select API Source ]
sourceConfiguration = open_api(config_folder, file_type="source")
if not sourceConfiguration:
print('cancel')
exit(0)
# [Select API Target ]
targetConfiguration = open_api(config_folder, file_type="target")
if not targetConfiguration:
print('cancel')
exit(0)
# [Merge Source and Target]
sourceConfiguration.update(targetConfiguration)
apiConfiguration = sourceConfiguration
# setup default environment
homeDev = HomeDevelopment().setup()
# [Create missing folders]
sourceDev = HomeDevelopment().setup()
targetDev = HomeDevelopment().setup()
# [Scan configuration for home, source, and target environment configurations]
for apiName in apiConfiguration:
if apiName == 'source':
# [Configure input sources from GIT repositories]
sourceDev = get_environment(apiConfiguration[apiName])
elif apiName == 'target':
# [Configure output targets from GIT repositories]
targetDev = get_environment(apiConfiguration[apiName])
print('=================')
report(homeDev, sourceDev, targetDev)
print('=================')
##############
# [Process Postgres Extentions]
##############
if targetDev.getFolder('db'):
if not Util().folder_exists(sourceDev.getFolder('db')):
# [Copy first set of _tasks code and config files]
Util().copyFolder(sourceDev.getFolder('db'),
targetDev.getFolder('db'))
else:
if not Util().confirm(
'* Install/Overwrite postgres code and extentions?', 'N'):
print(" Overwriting postgres configuration and extentions")
print(' - source ', sourceDev.getFolder('db'))
print(' - target ', targetDev.getFolder('db'))
# [Copy all files in extention/db folder]
for fn in Util().getFileList(sourceDev.getFolder('db')):
if Util().file_exists(targetDev.getFolder('db'), fn):
if not Util().confirm(' -- Overwrite {}?'.format(fn),
'N'):
Util().copy(
'{}/{}'.format(sourceDev.getFolder('db'), fn),
'{}/{}'.format(targetDev.getFolder('db'), fn))
else:
print(' - copy', fn)
Util().copy(
'{}/{}'.format(sourceDev.getFolder('db'), fn),
'{}/{}'.format(targetDev.getFolder('db'), fn))
#############
# [## Initalize docker-component]
#############
# [Define list of words to replace in docker-compose file]
replace_ = ['one_db', 'hapi-api']
replace_with = [targetDev.getName('db'), targetDev.getName('db_api')]
if not Util().file_exists(targetDev.getFolder('project'),
'docker-compose.test.yml'):
# [Copy docker-compose.yml to target when it doesnt exist]
#print('Create new docker-compose.yml')
dcDoc = Document(sourceDev.getFolder('project'),
'docker-compose.yml').load()
dcDoc.replace(replace_, replace_with)
dcDoc.saveAs(targetDev.getFolder('project'), 'docker-compose.test.yml')
else:
# [Ask to overwrite when docker-compose exists]
if not Util().confirm('* Overwrite docker-compose.test.yml?', 'N'):
dcDoc = Document(sourceDev.getFolder('project'),
'docker-compose.yml').load()
backup = Util().makeBackupFile(targetDev.getFolder('project'),
'docker-compose.test.yml')
print('backup', backup)
dcDoc = Document(sourceDev.getFolder('project'),
'docker-compose.yml').load()
dcDoc.replace(replace_, replace_with)
dcDoc.saveAs(targetDev.getFolder('project'),
'docker-compose.test.yml')
##############
# [## .env # if .env doesnt exist then create one]
#############
srcDoc = EnvironmentDocument(sourceDev.getFolder('env'), '.env').load()
srcDoc.replace(replace_, replace_with)
if not Util().file_exists(targetDev.getFolder('env'), '.env'):
# [Copy .env to target when it doesnt exist]
print('Create .env')
srcDoc.saveAs(targetDev.getFolder('env'), '.env')
else: # [if .env exists then update and add new variables]
if not Util().confirm('* Update .env?', 'N'):
# [Ask to update when .e nv exists]
print('Update .env')
trgtDoc = EnvironmentDocument(targetDev.getFolder('env'),
'.env').load()
#pprint(trgtDoc)
#print('--------')
trgtDoc.backup()
trgtDoc.update(srcDoc)
trgtDoc.save()
#trgtDoc.saveAs(targetDev.getFolder('env'), '.env')
#pprint(trgtDoc)
#############
# [## Process Static Scripts]
#############
# [Static scripts end with .static.sql]
if targetDev.getFolder('db'):
if not Util().confirm('* Install/Overwrite static scripts?', 'N'):
print("writing static scripts")
#############
# [Process Static Db-Api]
#############
process_documents_to(apiConfiguration, sourceDev, targetDev,
'db.sql', '00.db.sql')
#############
# [Process Static Database Scripts]
#############
process_documents_to(apiConfiguration, sourceDev, targetDev,
'table.sql', '10.base.table.sql')
#############
# [Process Base Function Scripts]
#############
# process_to_one(apiConfiguration, sourceDev, targetDev, 'base.function.sql','12.base.function.sql')
process_documents_to(apiConfiguration, sourceDev, targetDev,
'base.function.sql', '20.base.function.sql')
#############
# [Process Api Function Scripts]
#############
# process_to_one(apiConfiguration, sourceDev, targetDev, 'api.function.sql','20.api.function.sql')
# combine_documents(apiConfiguration, sourceDev, targetDev, 'api.function.sql', '30.api.function.sql')
process_documents_to(apiConfiguration, sourceDev, targetDev,
'api.usage.sql', '30.api.usage.sql')
#############
# [Process Static Test Scripts]
#############
#combine_documents(apiConfiguration, sourceDev, targetDev, 'base.test.sql', '90.base.test.sql')
#combine_documents(apiConfiguration, sourceDev, targetDev, 'api.test.sql', '92.api.test.sql')
#############
# [Process Static Data Scripts]
# retired the data scripts. data is now encpsulated in tests
#############
# process_to_one(apiConfiguration, sourceDev, targetDev, 'data.sql','80.data.sql')
#############
# [Process Static Cleaup Scripts]
# retired the data scripts. data is now encpsulated in tests
#############
# process_to_one(apiConfiguration, sourceDev, targetDev, 'cleanup.sql','98.test.cleanup.sql')
#############
# [Process DbApi]
#############
#Util().copyFolder(sourceDev.getFolder('db_api'), targetDev.getFolder('db_api'),ignore=shutil.ignore_patterns('node_modules'))
#
#############
# [## Process multiple API SQL Definitions]
# skip api-static, _tasks, source and target
#############
# [Set target _tasks name]
if targetDev.getFolder('db'):
apiNameList = [
nm for nm in apiConfiguration
if apiConfiguration[nm]['kind'] == 'api-definition'
]
for apiName in apiNameList:
apiScriptFilename = '{}.{}.api.function.sql'.format(
apiConfiguration[apiName]['prefix'],
apiConfiguration[apiName]['name'].upper())
# [Combine all API functions into one script]
combined_sql_list = []
combined_sql_list.append('-- api : {}'.format(
apiName.upper()))
combined_sql_list.append('-- schema : {}'.format(
apiConfiguration[apiName]['schema']))
combined_sql_list.append('-- generated on: {}'.format(
datetime.now()))
combined_sql_list.append('-- source project: {} '.format(
sourceDev.getName('project')))
#combined_sql_list.append('\c {}'.format(targetDev.getName('db')))
#combined_sql_list.append('SET search_path TO {};'.format(', '.join(apiConfiguration['_tasks']['schema'])))
# [Generate POST Function]
combined_sql_list.append('-- POST')
combined_sql_list.extend(PostTemplate(apiName, folder='../templates', filename='post.sql.template') \
.validate(apiConfiguration)\
.apply(apiConfiguration))
# [Generate GET Function]
combined_sql_list.append('-- GET')
combined_sql_list.extend(GetTemplate(apiName, folder='../templates', filename='get.sql.template') \
.validate(apiConfiguration) \
.apply(apiConfiguration))
# [Generate DELETE Function]
combined_sql_list.append('-- DELETE')
combined_sql_list.extend(DeleteTemplate(apiName, folder='../templates', filename='delete.sql.template') \
.validate(apiConfiguration) \
.apply(apiConfiguration))
# [Generate PUT Function]
combined_sql_list.append('-- PUT')
combined_sql_list.extend(PutTemplate(apiName, folder='../templates', filename='put.sql.template') \
.validate(apiConfiguration) \
.apply(apiConfiguration))
# [Assemble API (POST, GET, PUT, and Delete) Functions into single script]
newDoc = Document(targetDev.getFolder('scripts'),
apiScriptFilename).load(combined_sql_list)
# [Confirm overwrite of existing API files]
if not Util().file_exists(targetDev.getFolder('scripts'),
apiScriptFilename):
print(' - Writing API {} script'.format(apiScriptFilename))
# [Create Api when Api doesnt exist]
newDoc.write()
else:
if not Util().confirm(
'* Overwrite API {} script?'.format(apiScriptFilename),
'N'):
# [Confirm the overwrite of existing Api script]
# [Backup Api script before overwriting]
backup = Util().makeBackupFile(
targetDev.getFolder('scripts'), apiScriptFilename)
print(" - Overwriting API {} script".format(
apiScriptFilename))
newDoc.write()
apiStaticNameList = [
nm for nm in apiConfiguration
if apiConfiguration[nm]['kind'] == 'api-static'
]
#############
# [## Process static API scripts]
# Static scripts are copied, combined and renamed to the target folder
#############
if targetDev.getFolder('db'):
for apiName in apiStaticNameList:
sourceFilename = '{}.{}.{}.api.function.sql'.format(
'24', apiConfiguration[apiName]['schema'].replace('_', '.'),
apiConfiguration[apiName]['name'])
targetFilename = '{}.{}.api.function.sql'.format(
apiConfiguration[apiName]['prefix'],
apiConfiguration[apiName]['name'].upper())
combined_sql_list = []
combined_sql_list.append('-- api : {}'.format(
apiName.upper()))
combined_sql_list.append('-- schema : {}'.format(
apiConfiguration[apiName]['schema']))
combined_sql_list.append('-- copied on: {}'.format(datetime.now()))
combined_sql_list.append('-- source project: {}'.format(
sourceDev.getName('project')))
combined_sql_list.append('\c {}'.format(targetDev.getName('db')))
combined_sql_list.append('SET search_path TO {};'.format(', '.join(
apiConfiguration['_tasks']['schema'])))
# [Define list of words to replace in docker-compose file]
replace_ = ['one_db', 'hapi-api']
replace_with = [
targetDev.getName('db'),
targetDev.getName('db_api')
]
if not Util().file_exists(targetDev.getFolder('scripts'),
targetFilename):
# [Copy docker-compose.yml to target when it doesnt exist]
print('* Create new {}'.format(targetFilename))
print(' -- load {}'.format(apiName))
print(' -- swap out values ')
print(' -- write api to target ')
dcDoc = Document(sourceDev.getFolder('scripts'),
sourceFilename).load()
dcDoc.replace(replace_, replace_with)
dcDoc.saveAs(targetDev.getFolder('scripts'), targetFilename)
else:
# [Ask to overwrite when docker-compose exists]
if not Util().confirm('* Overwrite {}?'.format(targetFilename),
'N'):
#print('* Overwrite api-static')
print(' -- load {}'.format(apiName))
print(' -- replace values ')
print(' -- save api to target ')
dcDoc = Document(sourceDev.getFolder('scripts'),
sourceFilename).load()
backup = Util().makeBackupFile(
targetDev.getFolder('scripts'), targetFilename)
print(' -- backup', backup)
dcDoc = Document(sourceDev.getFolder('scripts'),
sourceFilename).load()
dcDoc.replace(replace_, replace_with)
dcDoc.saveAs(targetDev.getFolder('scripts'),
targetFilename)
###################
# [Write API tests]
##################
'''
from lib.util import Util
from bs4 import BeautifulSoup
import re
import time
#通用变量
totalPagePattern = re.compile(r'_PAGE_COUNT="(\d+)"')
totalPagePattern1 = re.compile(r'_(\d+)\.shtml')
articleFull = ''
caijiUtil = Util(rootPath='D:/www/data',
host='http://kaoyan.eol.cn',
downloadPath='/files/kaoyan1')
#从网址中获取采集到的数据 ，封装成字典数据返回
def caijiUrl(url):
if not url:
return None
content = caijiUtil.getUrlContent(url)
if not content:
return None
soup = BeautifulSoup(content, 'html5lib')
article = soup.select('.TRS_Editor')
article = article[0] if article else None
title = str(soup.select('.page_title')[0].text)
tags = soup.select('.n_left a')
tag = ''
#coding=utf-8
from bs4 import BeautifulSoup
import re, time, os, sys
sys.path.append("..")
from lib.dispatchUrl import DispatchUrl
from dal.articleDal import insertArticle
from lib.util import Util
#通用变量
totalPagePattern = re.compile(r'_PAGE_COUNT="(\d+)"')
totalPagePattern1 = re.compile(r'_(\d+)\.shtml')
articleFull = ''
caijiUtil = Util(rootPath='D:/www/data',
host='http://blog.csdn.net',
downloadPath='/files/blog0',
defaultExt='png')
#从网址中获取采集到的数据 ，封装成字典数据返回
def caijiUrl(url):
if not url:
return None
content = caijiUtil.getUrlContent(url)
if not content:
return None
soup = BeautifulSoup(content, 'html5lib')
article = soup.select('#article_content')
#coding=utf-8
import time, re
from bs4 import BeautifulSoup
from lib.dispatchUrl import DispatchUrl
from lib.util import Util
dispatchUrl = DispatchUrl('csdn')
caijiUtil = Util(host='http://blog.csdn.net/')
#博主列表的第一页
beginUrls = [
'http://blog.csdn.net/wcyoot/article/list/1',
]
# 获取所有的的列表页
pageList = []
# 所有的url集合
urlList = []
index = 0
for beginUrl in beginUrls:
html = caijiUtil.getUrlContent(beginUrl)
if not html:
print("获取首页列表失败:: " + beginUrl)
continue
soup = BeautifulSoup(html, "html5lib")
pageList_ = soup.select('#papelist a')
#print(pageList_[-1])
#exit()
if pageList_:
totalPagePattern = re.compile(r'(\d+)$')
def get_remove_link(self, request):
result = False
parameters = request.POST.dict()
#link = json.loads(parameters['link'])
#source = link['source']
#destination = link['target']
source_type = parameters['source_type'] # source['info']['type']
destination_type = parameters[
'target_type'] # destination['info']['type']
source_id = parameters['source']
destination_id = parameters['target']
group = parameters['group']
current_data = json.loads(self.data_project)
tosca_definition = Util().loadyamlfile(PATH_TO_TOSCA_DEFINITION)
removed = False
if 'requirements' in tosca_definition['node_types'][source_type]:
for req in tosca_definition['node_types'][source_type][
'requirements']:
for key in req:
if req[key]['node'] == destination_type:
if 'requirements' not in \
current_data['toscayaml'][group]['topology_template']['node_templates'][
source_id] or \
current_data['toscayaml'][group]['topology_template']['node_templates'][
source_id][
'requirements'] is None:
current_data['toscayaml'][group][
'topology_template']['node_templates'][
source_id]['requirements'] = []
requirements = \
current_data['toscayaml'][group]['topology_template']['node_templates'][source_id][
'requirements']
element = next(
(x for x in requirements if key in x.keys()), None)
if element is not None:
removed = True
requirements = \
current_data['toscayaml'][group]['topology_template']['node_templates'][source_id][
'requirements'].remove(element)
if not removed and 'requirements' in tosca_definition['node_types'][
destination_type]:
for req in tosca_definition['node_types'][destination_type][
'requirements']:
for key in req:
if req[key]['node'] == source_type:
if 'requirements' not in \
current_data['toscayaml'][group]['topology_template']['node_templates'][
destination_id] or \
current_data['toscayaml'][group]['topology_template']['node_templates'][
destination_id][
'requirements'] is None:
current_data['toscayaml'][group][
'topology_template']['node_templates'][
destination_id]['requirements'] = []
requirements = \
current_data['toscayaml'][group]['topology_template']['node_templates'][destination_id][
'requirements']
element = next(
(x for x in requirements if key in x.keys()), None)
if element is not None:
removed = True
requirements = \
current_data['toscayaml'][group]['topology_template']['node_templates'][destination_id][
'requirements'].remove(element)
self.data_project = current_data
# self.validated = validate #TODO(stefano) not clear if this is the validation for the whole project
self.update()
result = True
return result
from lib.util import Util
from bs4 import BeautifulSoup
import re, time
from dal.articleDal import insertArticle
from lib.dispatchUrl import DispatchUrl
#通用变量
totalPagePattern = re.compile(r'_PAGE_COUNT="(\d+)"')
totalPagePattern1 = re.compile(r'_(\d+)\.shtml')
articleFull = ''
caijiUtil = Util(rootPath='D:/www/data',
host='http://www.cnedu.cn',
downloadPath='/files/kaoyan0')
#从网址中获取采集到的数据 ，封装成字典数据返回
def caijiUrl(url):
if not url:
return None
content = caijiUtil.getUrlContent(url)
if not content:
return None
soup = BeautifulSoup(content, 'html5lib')
article = soup.select('#fontzoom')
article = article[0] if article else None
title = soup.select('.list-left h1')[0].text
filters = ['2016', '2017', '2015', '2014', '2013', '2012', '2011', '2010']
flag = False
def main():
combined_sql_list = [] # eventually is written to a file
# [Generate API sql file]
print('Generate README.md')
print(' - load api configuration, generate funcPattern key and values')
# get configuration file name {folder: "", name: ""}
# get list of files of type .json in folder ./config
# [Use a configuration file]
config_folder = '{}'.format(os.getcwd().replace('_documents', 'config'))
print('config_folder', config_folder)
# [Select API Source ]
sourceConfiguration = open_api(config_folder, file_type="source")
if not sourceConfiguration:
print('cancel')
exit(0)
# [Select API Target ]
targetConfiguration = open_api(config_folder, file_type="target")
if not targetConfiguration:
print('cancel')
exit(0)
# [Merge Source and Target]
sourceConfiguration.update(targetConfiguration)
apiConfiguration = sourceConfiguration
# setup default environment
homeDev = HomeDevelopment().setup()
# [Create missing folders]
sourceDev = HomeDevelopment().setup()
targetDev = HomeDevelopment().setup()
# [Scan configuration for home, source, and target environment configurations]
pprint(apiConfiguration)
for apiName in apiConfiguration:
if apiName == 'source':
# [Configure input sources from GIT repositories]
sourceDev = get_environment(apiConfiguration[apiName])
elif apiName == 'target':
# [Configure output targets from GIT repositories]
targetDev = get_environment(apiConfiguration[apiName])
print('=================')
#report(homeDev, sourceDev, targetDev)
print('=================')
pprint(sourceDev)
print(sourceDev.getFolder('group'))
script_list = [{
"source": fn,
"target": "README.{}.md".format(fn)
} for fn in Util().getFileList(sourceDev.getFolder('group'), ext='.sh')]
print('script_list', script_list)
for fn in script_list:
print('script', sourceDev.getFolder('group'), fn['source'])
print('script', targetDev.getFolder('group'), fn['target'])
scriptDoc = ScriptReadmeDocument(sourceDev.getFolder('group'),
fn['source']).load()
pprint(scriptDoc)
scriptDoc.saveAs(targetDev.getFolder('group'), fn['target'])
'''
from lib.hdfs import Hdfs
from lib.spark import SparkAutomate
from lib.hive import HiveAutomate
from lib.oozie import OozieAutomate
from lib.data import Dataset
from lib.util import Util
from conf.config import *
import subprocess
import time
if __name__ == '__main__':
util = Util()
util.get_cluster_details()
hdfs = Hdfs()
start_time = time.time()
try:
print("Setting up the datasets .. ")
data = Dataset( URL_LIST, util)
data.automate()
except Exception as e:
print("Failed to set up the datasets !")
print("Following exception has occured " + str(e))
else :
try :
print("Running spark application..")
spark = SparkAutomate(SPARK_DIR_LIST, util, hdfs)
spark.automate()
except Exception as e:
print("Failed to run spark application !")
def main():
clause2method = {'insert':'POST', 'update':'PUT', 'query':'GET', 'delete':'DELETE'}
combined_sql_list = [] # eventually is written to a file
# [Generate API sql file]
print('Generate API')
print(' - load api configuration, generate funcPattern key and values')
# get configuration file name {folder: "", name: ""}
# get list of files of type .json in folder ./config
# [Use a configuration file]
config_folder = '{}'.format(os.getcwd().replace('_tasks','config'))
# [Select API Source ]
sourceConfiguration = open_api(config_folder,file_type="source")
if not sourceConfiguration:
print('cancel')
exit(0)
# [Select API Target ]
targetConfiguration = open_api(config_folder,file_type="target")
if not targetConfiguration:
print('cancel')
exit(0)
# [Merge Source and Target]
sourceConfiguration.update(targetConfiguration)
apiConfiguration = sourceConfiguration
# setup default environment
homeDev = HomeDevelopment().setup()
# [Create missing folders]
sourceDev = HomeDevelopment().setup()
targetDev = HomeDevelopment().setup()
# [Scan configuration for home, source, and target environment configurations]
for apiName in apiConfiguration:
if apiName == 'source':
# [Configure input sources from GIT repositories]
sourceDev = get_environment(apiConfiguration[apiName])
elif apiName == 'target':
# [Configure output targets from GIT repositories]
targetDev = get_environment(apiConfiguration[apiName])
print('=================')
report(homeDev, sourceDev, targetDev)
print('=================')
print('=================')
print('Base Tests Combined')
print('=================')
#fileList = Util().getFileList(sourceDev.getDbFolder('sql'),'static.sql')
fileList = Util().getFileList(sourceDev.getFolder('scripts'),'base.test.sql')
fileList.sort()
# [Replace project specific values]
replace_ = ['one_db', 'hapi-api']
replace_with = [targetDev.getName('db'), targetDev.getName('db_api')]
targetName = '90.base.test.sql'
combinedDocument = Document(targetDev.getFolder('scripts'), targetName)
# [Backup the target file]
backup = Util().makeBackupFile(targetDev.getFolder('scripts'), targetName)
print(' - backup : {}'.format(backup))
# [Move Static Files to Target Folder]
for fileName in fileList:
# [Combine base tests into one file]
# [Copy from a source folder to a target folder]
staticDocument = Document(sourceDev.getFolder('scripts'), fileName) \
.load() \
.replace(replace_, replace_with)
combinedDocument.extend(staticDocument)
combinedDocument.save()
print('=================')
print('API Tests')
print('=================')
apiNameList = [nm for nm in apiConfiguration if apiConfiguration[nm]['kind'] == 'api-definition' or apiConfiguration[nm]['kind'] == 'api-static']
# [Move Static Files to Target Folder]
#targetNames = []
#sourceNames = []
names = []
src_folder = sourceDev.getFolder('scripts')
tmpl_folder = '../templates'
trg_folder = targetDev.getFolder('scripts')
clauses = ['insert','query','update','delete']
for apiName in apiNameList:
schema = apiConfiguration[apiName]['schema'].replace('_','.')
prefix = apiConfiguration[apiName]['prefix-test']
# make list of existing test files
names = [{"source":'{}.test.{}.{}.{}.api.test.sql'.format(prefix,schema, apiName.upper(), clause),
"target":'{}.{}.{}.api.test.sql'.format(prefix,apiName.upper(), clause),
"template": '{}.test.sql.template'.format(clause),
"clause": '{}'.format(clause),
"method": '{}'.format(clause2method[clause])} for clause in clauses]
names = [pair for pair in names if Util().file_exists(src_folder, pair['source'])]
#
for pair in names:
backup = None
#print('pair', pair, 'kind', apiConfiguration[apiName]['kind'])
if apiConfiguration[apiName]['kind'] == 'api-definition':
# handle backup
if Util().file_exists(trg_folder, pair['target']):
backup = Util().makeBackupFile(targetDev.getFolder('scripts'), pair['target'])
# Use Template
if Util().file_exists(tmpl_folder, pair['template']): # Use Template
# Use Template
print(' - generate test {} FROM {}'.format(pair['target'], pair['template']))
if backup: print(' -- backup ', backup)
templateDoc = TestTemplate(apiName, pair['method'], pair['template']) \
.apply(apiConfiguration)\
.saveAs(trg_folder, pair['target'])
elif Util().file_exists(src_folder, pair['source']): # Copy from existing file
# Copy from source
print(' - copy test {} FROM {} '.format(pair['target'],pair['source']))
# load
# replace
# save
doc = Document(src_folder, pair['source']) \
.load() \
.replace(replace_, replace_with)\
.saveAs(trg_folder,pair['target'])
else: # No test available
print('No Test Available for ', pair['source'])
elif apiConfiguration[apiName]['kind'] == 'api-static':
#print('template', tmpl_folder)
print(' - copy test for api-static ', pair['source'])
# [Backup the target file]
#print(' - backup : {}'.format(fileName))
apiNameList = [nm for nm in apiConfiguration if apiConfiguration[nm]['kind'] == 'api-static']
# [Move Static Files to Target Folder]
'''
