def main(bcbio_dir, bed, depth, threads=None, isdebug=True):
snp_file = verify_file(bed)
depth_cutoff = depth
log.init(isdebug)
try:
import az
except ImportError:
parallel_cfg = ParallelCfg(threads=threads)
else:
sys_cfg = az.init_sys_cfg()
parallel_cfg = ParallelCfg(
scheduler=sys_cfg.get('scheduler'),
queue=sys_cfg.get('queue'),
resources=sys_cfg.get('resources'),
threads=threads or sys_cfg.get('threads'),
tag='clearup')
log.info('Loading bcbio project from ' + bcbio_dir)
log.info('-' * 70)
proj = BcbioProject()
proj.load_from_bcbio_dir(bcbio_dir, proc_name='clearup')
log.info('Loaded ' + proj.final_dir)
log_dir = safe_mkdir(join(proj.log_dir, 'clearup'))
work_dir = safe_mkdir(join(proj.work_dir, 'clearup'))
out_dir = safe_mkdir(join(proj.date_dir, 'clearup'))
with parallel_view(len(proj.samples), parallel_cfg, log_dir) as parall_view:
genotype(proj.samples, snp_file, parall_view, work_dir, out_dir, proj.genome_build)
def make_makefile(platform, project):
## FIX pncrt madness
if project.getTargetType() in [ 'dll' , 'exe' ]:
fix_pncrt(project)
## Create Applescript
mprj = ProjectToMacCWProjectData(platform, project)
applescript_path = macfs.FSSpec(project.makefile_name).as_pathname()
ASMakefile(mprj).script.CompileAndSave(applescript_path)
## Create Uber XML file
generate_uberxml(platform, project, mprj)
## Create an applescript stub file for reading the Uber XML file
# this needs better handling/reporting of build errors
script = ascript.CreateAppleScript()
script.Append(
'tell application %s' % (os.environ["BUILD_SW"]),
' with timeout of 99999 seconds',
' try',
' make new (project document) as "%s" with data ("%s")' % (
macpath.join(os.getcwd(),project.makefile_name+"_uber.prj"),
macpath.join(os.getcwd(),project.makefile_name+"_uber.xml")),
' set cwErrorList to Make Project with ExternalEditor',
' Close Project',
' on error errText number errnum',
' return errText & return',
' end try',
' end timeout',
'end tell')
uber_ascript = macfs.FSSpec(project.makefile_name+"_uber").as_pathname()
script.CompileAndSave(uber_ascript)
## Pretend like everything went well
return None
def RunProject(self):
mprj = self.mprj
script = self.script
all = self.all
## create the project XML files to be imported into CW
WriteProjectXML(mprj)
xml_filename=macpath.join(os.getcwd(), mprj.project.makefile_name+".xml")
all.extend( [
' tell application %s' % (mprj.ide_path),
' with timeout of 99999 seconds',
' try',
' make new (project document) as "%s" with data ("%s")' \
% (mprj.project_file_path, xml_filename),
' set cwErrorList to Make Project with ExternalEditor',
' set myerrors to my ExtractCWErrors(cwErrorList)',
' Close Project',
' on error errText number errnum',
' try',
' Close Project',
' end try',
' --rethrow the error',
' return errText & return',
' end try',
' end timeout',
' end tell' ])
def Clean(self):
"""Emits a AppleScript function to clean the target, project file, and
project file data for a target."""
self.script.Append(
'-- Allow the items to be made clean',
'on Clean()',
)
for sumake in self.mprj.project.submakes:
m_path = macpath.normpath(macpath.join(os.getcwd(), sumake.makefile()))
self.script.Append(
' set scriptobj to (load script file "%s")' % (m_path),
' tell scriptobj',
' Clean()',
' end tell' )
if self.mprj.project.getTargetType() != "":
self.script.Append(
' tell application "Finder"',
' with timeout of 99999 seconds',
' if file "%s" exists then' % (self.mprj.project_file_path),
' delete file "%s"' % (self.mprj.project_file_path),
' end if',
' if folder "%s" exists then' % (self.mprj.project_data_path),
' delete folder "%s"' % (self.mprj.project_data_path),
' end if')
if self.mprj.rtarget:
self.script.Append(
' if file "%s" exists then' % (self.mprj.rproject_file_path),
' delete file "%s"' % (self.mprj.rproject_file_path),
' end if',
' if folder "%s" exists then' % (self.mprj.rproject_data_path),
' delete folder "%s"' % (self.mprj.rproject_data_path),
' end if')
self.script.Append(
' end timeout',
' end tell')
self.script.Append(
'end Clean',
'')
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from macpath import dirname, join
import sklearn
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
path = join(dirname(__file__), "california_housing_train.csv")
data = pd.read_csv(path)
data = data.values
X = np.c_[data[:, 1], data[:, 2], data[:, 3], data[:, 4]]
Y = np.c_[data[:, 0]]
plt.scatter(X[:, 0], Y, s=10, color='g', marker='x')
plt.title(" housing_meidan_age and median_house_value", fontsize=14)
plt.xlabel('housing_median_age', fontsize=14)
plt.ylabel('median_house_value', fontsize=10)
plt.grid(True)
plt.show()
plt.scatter(X[:, 1], Y, s=10, color='g', marker='x')
plt.title(" total_rooms and median_house_value", fontsize=14)
plt.xlabel('total_rooms', fontsize=14)
plt.ylabel('median_house_value', fontsize=10)
plt.grid(True)
plt.show()
def __init__(self, mprj):
self.mprj = mprj
## start the script and define subroutines
script = ascript.CreateAppleScript()
self.script = script
all = []
self.all = all
for sumake in mprj.project.submakes:
m_path = macpath.normpath(macpath.join(os.getcwd(), sumake.makefile()))
all.extend( [
'set scriptobj to (load script file "%s")' % (m_path),
'tell scriptobj',
' set myerrors to myerrors & return & all()',
'end tell' ] )
script.Extend(mprj.project.pre_target_buff)
self.Clean()
if mprj.project.getTargetType() != "":
WritePrefixFile(mprj)
WriteResourcePrefixFile(mprj)
WriteExportFile(mprj)
self.VerifyPath()
self.ExtractCWErrors()
if len(mprj.weak_link_list):
self.SetWeakLink()
if len(mprj.post_build_script):
self.PostBuildScript()
if mprj.rtarget:
self.DefineResourceProject()
## FIXME, only do this for if asked to 'clean' build
all.extend( [
' --clean out old project and data by default',
' Clean()' ])
## write the "all" function, like "make all" in a Makefile
all.extend( [
' --make the output directory, and target directory',
' verifypath("%s")' % (mprj.output_dir_path),
' verifypath("%s")' % (mprj.target_dir_path) ] )
if mprj.rtarget:
all.extend( [
'--build the windows resource dll',
'myerrors = myerrors & return & ResourceProject()' ] )
self.RunProject()
## for DLL target types, we make a alias to the dll with a .lib
## extention so developers can do implicit linking to DLL's the
## same way they do on Windows
##
## Note: when Windows compiles a DLL, it creates a companion .LIB
## library with the same name; you link to the DLL by linking
## in the stub .LIB
absolute_output_dir = os.path.join(os.getcwd(), mprj.output_dir)
absolute_output_path = os.path.join(absolute_output_dir, mprj.output_name)
if mprj.target_type == "dll":
if mprj.project.opt_target_name:
alias_name= mprj.project.opt_target_name
else:
alias_name= mprj.target_name
alias_name = "%s.LIB" % (string.upper(alias_name))
absolute_alias_path = os.path.join(absolute_output_dir, alias_name)
all.extend( [
'tell application "Finder"',
' try',
' if (file "%s") exists then' % (absolute_alias_path),
' else',
' make new alias file to (file "%s") at (folder "%s") '\
' with properties {name:"%s"}' % (
absolute_output_path, absolute_output_dir, alias_name),
' end if',
' on error',
' end try',
'end tell' ])
if len(mprj.post_build_script):
all.extend( [
'--execute the custom subroutine',
'DoPostBuild()' ])
## copy the built target and finish off the script
all.extend( [
'-- Copy results to common output folder',
'tell application "Finder"',
' if (file "%s") exists then' % (absolute_output_path),
' Duplicate file "%s" to folder "%s" with replacing' % (
absolute_output_path, mprj.target_dir_path),
' end if',
'end tell' ])
if len(all):
script.Append('on all()',
'set myerrors to ""')
script.Extend(all)
script.Append(
'return myerrors',
'end all',
'-- run the "all()" function',
'return all()')
script.Extend(mprj.project.post_target_buff)
## for DRM signing
if mprj.project.getTargetType() == "dll" and \
mprj.project.BuildOption("drmsign") and \
mprj.project.CheckDRMSign():
import shell
shell.mkdir(os.path.dirname(absolute_output_path))
open(absolute_output_path+"--drmsign","w").write("signme!")
'''
macpath 模块
macpath 模块( 参见 Example 13-2 )提供了 Macintosh 平台下的 os.path 功能.
你也可以使用它在其他平台处理 Macintosh 路径.
'''
import macpath
file = 'my:little:pony'
print("isabs", "=>", macpath.isabs(file))
print("dirname", "=>", macpath.dirname(file))
print("basename", "=>", macpath.basename(file))
print("normpath", "=>", macpath.normpath(file))
print("split", "=>", macpath.split(file))
print("join", "=>", macpath.join(file, "zorba"))
'''
isabs => True
dirname => my:little
basename => pony
normpath => my:little:pony
split => ('my:little', 'pony')
join => my:little:pony:zorba
'''
"""
os.environ["COLUMNS"] = str(columns())
title = "%s" % " ".join(usrcmdline)
pad = "#" * ((columns() - len(title)) / 2 - 1)
print pad + " " + title + " " + pad
# ===-----------------------------------------------------------------------===
# perform a dry run to build the command line arguments
# ===-----------------------------------------------------------------------===
env = os.environ.copy()
env["FUZZBALL_DRY_RUN"] = "1"
t1 = time.time()
print "Now in run-emu-fuzzball.py. Performing dry run...\n",
sys.stdout.flush()
print join(usrcmdline)
p = subprocess.Popen(usrcmdline, stdout=subprocess.PIPE, env=env)
out, err = p.communicate()
assert p.wait() == 0, "%s %s\n" % (str(p.returncode), err)
if os.getenv("FUZZBALL_DRY_RUN_DEBUG", False):
print "#" * columns()
print out,
print "done (%.3fs)" % (time.time() - t1)
# ===-----------------------------------------------------------------------===
# parse the output produced by the dry run
# ===-----------------------------------------------------------------------===
print out
for l in out.split("\n"):
# if l.startswith("FUZZBALL_SYMBOLIC_VARIABLE"):
# print l
"""
parser.add_argument('-p', '--port', type=int, dest='ui_port', action='store', default=3333,
help='TCP port on which the GUI will be available (default: 3333')
return parser.parse_args(arguments)
if __name__ == '__main__':
# Parse command line arguments
arguments = parseArgs(argv[1:]) # Ignore script name
# Get the path of the project directory
PROJECT_DIR = dirname(abspath(__file__))
# Create database files directory
try:
mkdir(join(PROJECT_DIR, 'database', 'db_files'))
except:
pass # Repository exists
# Start GUI
if not arguments.no_gui:
wake_gui = WakeGui(args=(PROJECT_DIR, arguments.ui_port, arguments.debug,))
wake_gui.start()
# Start proxy core
if not arguments.no_core:
pass
# Run WAKe
stop_running = Event()
try:
def get_tens(iter):
global cargo
while 1:
print "\nTENS State: ",
while jump_to(cargo) == 'TENS':
print "#%2.1f " % cargo,
cargo = iter.next()
yield (jump_to(cargo), cargo)
def exit(iter):
jump = raw_input('\n\n[co-routine for jump?] ').upper()
print "...Jumping into middle of", jump
yield (jump, iter.next())
print "\nExiting from exit()..."
sys.exit()
def toJSON(input):
reader = Reader(input)
parser = Parser(reader, asJson=True)
return json.dumps(parser.runtime)
# num_stream = math_gen(1)
# cargo = num_stream.next()
# gendct = {'ONES': get_ones(num_stream),'TENS': get_tens(num_stream), 'OUT_OF_RANGE': exit(num_stream)}
# scheduler(gendct, jump_to(cargo))
if __name__ == "__main__":
DIR = dirname(abspath(__file__))
TOMLFiles = glob(join(DIR, '*.toml'))
if __name__ == '__main__':
for filename in TOMLFiles:
with open(filename) as file:
print toJSON(file)
"""
def parse(self, response):
mycsv = [
'NULL', 'NULL', 'NULL', 'NULL', 'NULL', "NULL", 'NULL', 'NULL',
'NULL'
] # 初始化提取信息列表
# 爬取图片url
imgurl = ""
img = response.xpath(
'//body//div[@class="t_fsz"]//ul/li//ignore_js_op/img/@file'
).extract()
if len(img) > 0:
#print(img, "图片")
#这里拉了多张图片
for s in img:
#str = img[0]
rs = s.split("forum")
tem = "https://youtu.baobeihuijia.com/forum" + rs[1]
if imgurl == "":
imgurl = tem
else:
from macpath import join
imgurl = join(",".join((imgurl, tem)))
#print(rs[1])
else:
imgurl = "http://www.svmuu.com/img/4.0/backstage/no-data.jpg"
pass
#print(imgurl)
#爬取文字信息
for li in response.xpath('//body//div[@class="t_fsz"]//ul/li'):
# 伪装
from scrapy.conf import settings
origin = "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/"
ran = random.randint(10, 100000)
origin += str(ran)
settings['BOT_NAME'] = origin
info = li.xpath('font/text()').extract()
#print(info)
if (info != None):
for item in info:
if "失踪日期" in item[0:10] or "失踪日期" in item[0:10]:
index = item.find("：")
missingdate = item[index + 1:]
#正则化匹配
date_reg_exp = re.compile(
'\d{4}[-/年]\d{1,2}[-/月]\d{2}\s*')
matches_list = date_reg_exp.findall(missingdate)
#for match in matches_list:
match = None
if len(matches_list) == 0:
#print("正则表达式1无匹配"+missingdate+"，尝试2")
date_reg_exp1 = re.compile(
'\d{4}[-/.,年]\d{1,2}[-/.,月]\s*')
matches_list = date_reg_exp1.findall(missingdate)
if len(matches_list) == 0:
#print("正则表达式2无匹配"+missingdate+"，尝试3")
date_reg_exp2 = re.compile('\d{4}[年]\s*')
matches_list = date_reg_exp2.findall(
missingdate)
if len(matches_list) == 0:
#print(missingdate+"无匹配")
match = None
else:
#print(matches_list[0])
match = matches_list[0]
times = None
#解析日期
if match != None:
try:
times = time.strptime(match, "%Y年%m月%d")
except ValueError:
try:
times = time.strptime(match, "%Y-%m-%d")
except ValueError:
try:
times = time.strptime(
match, "%Y/%m/%d")
except ValueError:
try:
times = time.strptime(
match, "%Y年%m月%d日 ")
except ValueError:
try:
times = time.strptime(
match, "%Y-%m-%d ")
except ValueError:
try:
times = time.strptime(
match, "%Y/%m/%d ")
except ValueError:
#print("错误的日期格式或日期"+match)
times = None
mycsv[8] = times
#print(missingdate)
pass
if "姓 名" in item[0:6]:
index = item.find("：")
missingName = item[index + 1:]
mycsv[0] = missingName
if "性 别" in item:
index = item.find("：")
sex = item[index + 1:]
mycsv[1] = sex
if "出生日期" in item:
index = item.find("：")
birth = item[index + 1:]
#正则化匹配
date_reg_exp = re.compile(
'\d{4}[-/年]\d{1,2}[-/月]\d{2}\s*')
matches_list = date_reg_exp.findall(birth)
#for match in matches_list:
match = None
if len(matches_list) == 0:
#print("正则表达式1无匹配"+birth+"，尝试2")
date_reg_exp1 = re.compile(
'\d{4}[-/年]\d{1,2}[-/月]\s*')
matches_list = date_reg_exp1.findall(birth)
if len(matches_list) == 0:
#print("正则表达式2无匹配"+birth+"，尝试3")
date_reg_exp2 = re.compile('\d{4}[年]\s*')
matches_list = date_reg_exp2.findall(birth)
if len(matches_list) == 0:
#print(birth+"无匹配")
match = None
else:
#print(matches_list[0])
match = matches_list[0]
times = None
#解析日期
if match != None:
try:
times = time.strptime(match, "%Y年%m月%d")
except ValueError:
try:
times = time.strptime(match, "%Y-%m-%d")
except ValueError:
try:
times = time.strptime(
match, "%Y/%m/%d")
except ValueError:
try:
times = time.strptime(
match, "%Y年%m月%d日 ")
except ValueError:
try:
times = time.strptime(
match, "%Y-%m-%d ")
except ValueError:
try:
times = time.strptime(
match, "%Y/%m/%d")
except ValueError:
#print("错误是日期格式或日期")
times = None
#print(times)
#print(birth)
mycsv[2] = times
if "失踪时身高" in item:
index = item.find("：")
tall = item[index + 1:]
print("string:" + tall)
date_reg_exp = re.compile('\d{1,4}')
matches_list = date_reg_exp.findall(tall)
#print(matches_list)
matchTall = None
if len(matches_list) == 0:
print("身高正则匹配失败 string:" + tall)
else:
matchTall = matches_list[0]
print("正则化字符串： " + matchTall)
mycsv[3] = matchTall
if "失踪地点" in item:
index = item.find("：")
adress = item[index + 1:]
mycsv[4] = adress
mycsv[5] = getBlnglat(adress)
if "失踪者特征描述" in item:
index = item.find("：")
describe = item[index + 1:]
mycsv[6] = describe
pattern = re.compile(r'^\d+')
detail = pattern.match(item)
if detail:
tem = mycsv[7]
if tem == "NULL":
tem = ""
s = tem + item
mycsv[7] = s
#and mycsv[1] != "NULL" and mycsv[2] != "NULL"
#整合信息
if mycsv[0] != "NULL" and mycsv[4] != "NULL" and mycsv[5] != "NULL":
print(mycsv)
item = PersonItem() # 实例化item类
item['name'] = mycsv[0]
item['sex'] = mycsv[1]
item['birth'] = mycsv[2]
item['tall'] = mycsv[3]
item['adress'] = mycsv[4]
item['cor'] = mycsv[5]
item['describe'] = mycsv[6]
item['other'] = mycsv[7]
item['imgsrc'] = imgurl
item['missingdate'] = mycsv[8]
yield item
'''
macpath 模块
macpath 模块( 参见 Example 13-2 )提供了 Macintosh 平台下的 os.path 功能.
你也可以使用它在其他平台处理 Macintosh 路径.
'''
import macpath
file = 'my:little:pony'
print("isabs", "=>", macpath.isabs(file))
print("dirname", "=>", macpath.dirname(file))
print("basename", "=>", macpath.basename(file))
print("normpath", "=>", macpath.normpath(file))
print("split", "=>", macpath.split(file))
print("join", "=>", macpath.join(file, "zorba"))
'''
isabs => True
dirname => my:little
basename => pony
normpath => my:little:pony
split => ('my:little', 'pony')
join => my:little:pony:zorba
'''
def WriteProjectSettingsXML(mprj, templateName):
""" Writes out the Applescript which writes the <SETTINGS> section of
the CodeWarrior XML. This includes all the CW project preferences,
source file list and link order. This function is used to write
out the settings for 'project.xml' and 'project_uber.xml' """
template_filename = os.path.join(os.environ['BUILD_ROOT'],"bin","mac", templateName)
template_text = open(template_filename,"r").read()
## set access paths
user_list = []
if templateName == "project_uber.xml":
# uber can't have local paths eg. ':', ':pub:'
# they must be like '::pndebug:', '::pndebug:pub:'
module_path = os.getcwd()
src_root_path = macpath.normpath(os.path.join(module_path,mprj.project.src_root_path))+":"
for (path, recursive, origin) in mprj.user_paths:
umake_lib.debug("USER_PATH: %s => (%s)" %
(repr(path),
repr(mprj.project.src_root_path)))
path = macpath.normpath(macpath.join(module_path, path))
if path[:len(src_root_path)] == src_root_path:
path = "#SRC_ROOT_PATH#" + path[len(src_root_path):]
umake_lib.debug("USER_PATH: => %s (%s)" %
(repr(path),
repr(src_root_path)))
user_list.append(path)
else:
for (path, recursive, origin) in mprj.user_paths:
path = macpath.normpath(path)
user_list.append(path)
## Set CodeWarrior prefs
empty_list = []
template_text = SetAccessPathBlock(template_text,
user_list,
empty_list,
empty_list,
"#USER_SEARCH_PATHS#")
system_list = []
recursive_list = []
origin_list = []
for (path, recursive, origin) in mprj.system_paths:
system_list.append(path)
recursive_list.append(recursive)
origin_list.append(origin)
template_text = SetAccessPathBlock(template_text,
system_list,
recursive_list,
origin_list,
"#SYSTEM_SEARCH_PATHS#")
## add files
file_list = string.join(mprj.source_list, ',')
source_dir,output_dir = macpath.split(mprj.output_dir_path)
target_type = ''
if output_dir == "debug":
target_type = "debug"
template_text = SetFileListBlock(template_text,
file_list,
"#TEXT_FILE_LIST#",
"Text",
target_type)
source_list = []
for file_name in mprj.source_list:
source_list.append(file_name)
if len(mprj.library_list):
library_list = string.join(mprj.library_list, ',')
template_text = SetFileListBlock(template_text,
library_list,
"#LIB_FILE_LIST#",
"Library",
"")
# add libs to source list since they need to be
# included with groups, link order iterms
for library in mprj.library_list:
lib_path, lib_name = os.path.split(library)
if lib_name not in mprj.source_list:
source_list.append(lib_name)
else:
template_text=string.replace(template_text, "#LIB_FILE_LIST#", "")
# link order
file_list = string.join(source_list, ',')
gLinkOrderBlockString="""
<FILEREF>
<PATHTYPE>Name</PATHTYPE>
<PATH>#THE_VALUE#</PATH>
<PATHFORMAT>MacOS</PATHFORMAT>
</FILEREF>
"""
template_text = SetPreferenceBlock(template_text,
file_list,
gLinkOrderBlockString,
"#LINK_ORDER_ITEMS#")
## add frameworks
if len(mprj.project.sys_frameworks):
framework_string_list = string.join(mprj.project.sys_frameworks, ',')
template_text=SetFrameworkBlock(template_text,
framework_string_list,
"#FRAMEWORK_LIST#")
else:
template_text=string.replace(template_text, "#FRAMEWORK_LIST#", "")
## group order
template_text=SetGroupBlock(template_text,
file_list,
"#GROUP_LIST_ITEMS#",
mprj.cwtarget_name)
template_text = string.replace(template_text,
"#GROUP_NAME#",
mprj.cwtarget_name)
## CW project preferences
template_text=SetPreferenceValue(template_text, "MWFrontEnd_C_prefixname", mprj.prefix_file)
template_text=SetPreferenceValue(template_text, "MWRez_Language_prefixname", mprj.rprefix_file)
## set remaining preferences - theses are defined in macos-carbon-powerpc-cw6.cf
for (panel, pref) in mprj.preferences.items():
for (pref_key, pref_value) in pref.items():
template_text = SetPreferenceValue(template_text,
pref_key,
pref_value)
# set target dir to debug/release
if templateName == "project_uber.xml":
module_path = mprj.project_file_path[:-1]
module_path, module_name = os.path.split(module_path)
module_path, module_name = os.path.split(module_path)
template_text = string.replace(template_text,
"#TARGET_DIR#",
"#SRC_ROOT_PATH#%s:%s:" % (module_name,output_dir))
template_text = string.replace(template_text,
"#OUTPUT_FILE_NAME#",
"%s:%s" % (output_dir,mprj.output_name))
else:
template_text = string.replace(template_text,
"#TARGET_DIR#",
":%s:" % (output_dir))
template_text = string.replace(template_text,
"#TARGET_NAME#",
mprj.cwtarget_name)
return template_text
