def main():
# read parameters
parser = argparse.ArgumentParser(prog='Log file parser')
log_parameter = parser.add_argument_group('Log parameters', 'Parameters are used for logging')
log_parameter.add_argument('--log_folder_path', default='logs', help='The log output position.')
log_parameter.add_argument('--log_level', default='DEBUG',
choices=['DEBUG', 'INFO', 'ERROR', 'WARNING', 'CRITICAL'], help='The log level.')
process_parameter = parser.add_argument_group('Process parameters', 'Parameters are used for main process')
# /data / web_analytics_log / 2016
process_parameter.add_argument('--input_folder_path', help='The folder contains log files.')
# /data0/weiboyi/azkaban/MSE/log/log_parse_to_info/2016
process_parameter.add_argument('--output_folder_path', help='The folder stores the results.')
process_parameter.add_argument('--output_file_name', default='pages_and_tracks_', help='The file name.')
process_parameter.add_argument('--output_file_type', default='.tsv', help='The file type.')
args = parser.parse_args()
set_logging(args.log_folder_path, args.log_level)
logger = logging.getLogger()
logger.info('Begin the process')
try:
process(args.input_folder_path, args.output_folder_path,
args.output_file_name, args.output_file_type)
except Exception as e:
logger.error(e)
raise
logger.info('Process end')
def do_go(self, rest):
if not rest:
self.character.brain.to_client.append("OK, but go where?")
elif rest in ["north", "south", "east", "west", "up", "down"]:
exit_found = False
for an_exit in self.character.room().exits:
if rest == an_exit[0]:
exit_found = True
this_message = self.character.name + " leaves to the " + rest + "."
SimpleStim(STIM_VISUAL, this_message, False, [self.character.room()], [self.character])
new_room = self.character.room().resolve_exit(an_exit)
self.character.move_to(new_room, new_room.contents)
process(self.character, "look")
# Improve this, if you care to, with a direction of entry.
this_message = self.character.name + " has entered the room."
SimpleStim(STIM_VISUAL, this_message, False, [self.character.room()], [self.character])
break
if not exit_found:
self.character.brain.to_client.append("You can't got that way from here.")
else:
self.character.brain.to_client.append("I don't understand that location or direction.")
def sound():
SOUNDMENU = pygame.image.load('../media/images/soundmenu.jpg')
SCREEN.blit(SOUNDMENU, (0, 0))
i = 0
while True:
process()
pygame.display.update()
clicked = pygame.mouse.get_pressed()
pos = pygame.mouse.get_pos()
if clicked[0] is 1:
if 875 < pos[0] < 917 and 25 < pos[1] < 54:
print('1')
break
elif 360 < pos[0] < 600 and 175 < pos[1] < 230:
print('1')
pygame.mixer.music.play(-1)
elif 360 < pos[0] < 600 and 270 < pos[1] < 320:
print('1')
pygame.mixer.music.pause()
CLOCK.tick(FPS)
def getText(self):
server, okPressed = QtWidgets.QInputDialog.getText(self.centralwidget, "服务时间", "请输入该线程的服务时间：", QtWidgets.QLineEdit.Normal, "")
if okPressed and server.isdigit():
if self.listWidget.count() == 1:
# self.pushButton.setVisible(True)
# self.pushButton0.setVisible(True)
global startTime
startTime = time.time()
first = NUM
second = int(time.time() - startTime)
thrid = int(server)
Process = process.process(NUM, second, int(server))
# 线程处理数据
self.apply = multiply.applyRound(True, 0,self)
self.apply.dictP[NUM] = Process
self.apply.start()
# 启动画图
self.on_start()
else:
first = NUM
second = self.apply.allTime
thrid = int(server)
print('新的线程加入队列...')
Process = process.process(NUM, second, int(server))
self.apply.hasSomeProcessCome = True
self.apply.dictP[NUM] = Process
s = (f' {str(first)}\t{str(second)}\t {str(thrid)}')
self.listWidget.addItem(s)
def main():
"""It's pretty simple at the moment. Just grab that, mutate it, and shove it back"""
# These paths are relative to the root dir
old_path = './previous.png'
new_path = './output.png'
process.process(old_path, new_path)
def main():
# read parameters
parser = argparse.ArgumentParser(prog='Hdfs source to hive')
log_parameter = parser.add_argument_group('Log parameters', 'Parameters are used for logging.')
log_parameter.add_argument('--log_folder_path', default='logs', help='The log output position.')
log_parameter.add_argument('--log_level', default='DEBUG',
choices=['DEBUG', 'INFO', 'ERROR', 'WARNING', 'CRITICAL'], help='The log level.')
group_call = parser.add_argument_group('Group call', 'Parameters are used for calling this plugin.')
group_call.add_argument('--hadoop_cmd_env', default='/usr/local/hadoop/bin/hadoop',
help='The hadoop path which will be to used to run.')
group_call.add_argument('--hive_cmd_env', default='/usr/local/apache-hive-XXX/bin/hive',
help='The hive path which will be to used to run.')
group_call.add_argument('--hive_conf_path',
default="XXX",
help='The conf path which will be to used to update table.')
group_call.add_argument('--hive_store_folder_path', default='/user/hive/warehouse/',
help='The hadoop path which will be to used to run.')
args = parser.parse_args()
set_logging(args.log_folder_path, args.log_level)
logger = logging.getLogger()
logger.info('Begin update')
try:
process(args.hadoop_cmd_env, args.hive_cmd_env, args.hive_conf_path, args.hive_store_folder_path)
except Exception, e:
logger.error(e.message)
def pikachu(imgName=""):
if request.method == 'POST':
# 检查是否允许上传的文件类型
if 'file' not in request.files:
flash('No file part')
print("NOT ALLOW")
return redirect(request.url)
file = request.files['file']
# 如果用户没有上传图片，返回一个空的filename
# 没有上传图片：
if file.filename == '':
flash('No selected file')
print("NO FILE")
return redirect(request.url)
if file and allowed_file(file.filename):
# 当前文件所在路径
upload_path = os.path.join(basepath, 'static/photo',
secure_filename(file.filename))
# 注意：没有的文件夹一定要先创建，不然会提示没有该路径
file.save(upload_path)
#g.ok=True
process.process(file.filename)
return render_template("page.html", imgName=file.filename)
#render_template("template.html",imgName=file.filename)
#return redirect(url_for("show_pkq",imgName=file.filename))
return render_template("page.html")
def api_predict():
start_time = time.time()
image1 = request.files.get('image1')
image2 = request.files.get('image2')
if image1 and image2:
try:
image1, image2 = image1.read(), image2.read()
result = process(image1, image2)
except Exception as ex:
result = {'error': str(ex)}
print(traceback.print_exc())
elif request.data is not None and isinstance(request.data, list):
try:
image1 = cv2.imdecode(np.fromstring(request.data[0], np.uint8),
cv2.IMREAD_COLOR)
image1 = cv2.cvtColor(image1, cv2.COLOR_BGR2RGB)
image2 = cv2.imdecode(np.fromstring(request.data[1], np.uint8),
cv2.IMREAD_COLOR)
image2 = cv2.cvtColor(image2, cv2.COLOR_BGR2RGB)
result = process(image1, image2)
except Exception as ex:
result = {'error': str(ex)}
print(traceback.print_exc())
else:
ex = 'Please post request.data as list of 2 encoded image as string.'
result = {'error': ex}
print(traceback.print_exc())
result['run_time'] = time.time() - start_time
print(result)
# return the data dictionary as a JSON response
return jsonify(result)
def resume():
RESUME = pygame.image.load('../media/images/resume.gif')
SCREEN.blit(RESUME, (0, 0))
while True:
process()
pygame.display.update()
clicked = pygame.mouse.get_pressed()
pos = pygame.mouse.get_pos()
if clicked[0] is 1:
if 300 < pos[0] < 450 and 310 < pos[1] < 350:
mainmenu()
break
elif 520 < pos[0] < 660 and 310 < pos[1] < 350:
break
CLOCK.tick(FPS)
def show_pkq(imgName):
if True:
process.process(imgName)
#g.ok = False
return render_template("Untitled-2.html", img=imgName)
else:
print("gok==false")
return redirect(url_for('index'))
def run_meta(filename, verbose=False):
raw_paths = retrieve_imgs(raw_data_dir, filename)
# Process raw data
iter_name_imgs = iterator_name_imgs(raw_paths)
process(iter_name_imgs, proc_data_dir, filename, verbose=verbose)
proc_paths = retrieve_imgs(proc_data_dir, filename)
iter_imgs = iterator_imgs(proc_paths)
activations = get_activations(list(iter_imgs), 1.96, verbose=verbose)
cbma_dict = dict_from_activations(activations, 119)
ibma_dict = dict_from_paths(proc_paths, 119)
if verbose:
print(cbma_dict)
print(ibma_dict)
# exit()
res_ALE, p_ALE = meta_ALE.fit(cbma_dict)
res_KDA, p_KDA = meta_KDA.fit(cbma_dict)
res_MKDA, p_MKDA = meta_MKDA.fit(cbma_dict)
res_Stouffers, p_Stouffers = meta_Stouffers.fit(ibma_dict)
res_Fishers, p_Fishers = meta_Fishers.fit(ibma_dict)
# Threshold meta-analysis maps
res_ALE, = fdr_threshold([res_ALE], p_ALE)
res_KDA, = fdr_threshold([res_KDA], p_KDA)
res_MKDA, = fdr_threshold([res_MKDA], p_MKDA)
res_Stouffers, = fdr_threshold([res_Stouffers], p_Stouffers)
res_Fishers, = fdr_threshold([res_Fishers], p_Fishers)
# Turn thresholded maps into bool maps
bool_ALE = img_utils.to_bool(res_ALE)
bool_KDA = img_utils.to_bool(res_KDA)
bool_MKDA = img_utils.to_bool(res_MKDA)
bool_Stouffers = img_utils.to_bool(res_Stouffers)
bool_Fishers = img_utils.to_bool(res_Fishers)
# Compare meta-analysis result
print(compare.kappa(bool_ALE, bool_KDA))
print(compare.kappa(bool_ALE, bool_MKDA))
print(compare.kappa(bool_KDA, bool_MKDA))
print(compare.kappa(bool_ALE, bool_Stouffers))
print(compare.kappa(bool_ALE, bool_Fishers))
exit()
plotting.plot_stat_map(res_ALE, title=f'{filename} ALE')
plotting.plot_stat_map(res_KDA, title=f'{filename} KDA')
plotting.plot_stat_map(res_MKDA, title=f'{filename} MKDA')
plotting.plot_stat_map(res_Stouffers, title=f'{filename} Stouffers')
plotting.plot_stat_map(res_Fishers, title=f'{filename} Fishers')
plt.show()
def main(config_path): if config_path == None: config_path = 'config/config.yml' parseConfig(config_path) global DOWNLOAD_PATH DOWNLOAD_PATH = os.path.join( basePath , config["download"]["folder"] + "/") PROCESSED_PATH = os.path.join( basePath , config["processed"]["folder"] + "/") list=s3.list_objects(Bucket=config["s3"]["buckets"]["enriched"]["good"]["bucket"], Prefix=(config["s3"]["buckets"]["enriched"]["good"]["path"] + "/"))['Contents'] downloadFiles( list ) process(in_dir=DOWNLOAD_PATH, out_dir=PROCESSED_PATH)
def main():
parma = {
'data': [
{
'executeParam':
"""{"func_name": null, "region_code": "171800", "page_num": null, "region_name": "三门峡", "func_code": null}"""
},
]
}
process.process(parma)
def game_fn (num, adder):
SCREENWIDTH, SCREENHEIGHT = 640, 360
screen = pygame.display.set_mode((640, 360),0, 32)
WIDTH, HEIGHT = 630, 360
screen = pygame.display.set_mode((WIDTH, HEIGHT))
background = pygame.image.load("basement.jpg")
clock = pygame.time.Clock()
FPS = 24
exterminator = Exterminator(0, SCREENHEIGHT - 40, "Exterminator1.png", 0)
total_frames = 0
end_game = True
p = 0
x = 0
while end_game:
process(exterminator, FPS, total_frames, adder)
exterminator.motion(SCREENWIDTH, SCREENHEIGHT)
Fire.update_all(SCREENWIDTH, SCREENHEIGHT)
FireProjectile.movement()
total_frames = total_frames + 1
screen.blit(background, (0, 0))
BaseClass.allsprites.draw(screen)
FireProjectile.List.draw(screen)
score_stat = score(num, SCREENHEIGHT)
p = score_stat
num = p
if p == 10:
end_game = False
elif p == 20:
end_game = False
elif p == 30:
end_game = False
elif p == 40:
end_game = False
for fire in Fire.List:
col = pygame.sprite.spritecollide(fire, Exterminator.List, True)
if len(col) > 0:
end_game = False
font = pygame.font.Font(None, 36)
text = font.render("Kills: " + str(num), 1, (10, 10, 10))
screen.blit(text, (443, 7))
clock.tick(FPS)
pygame.display.flip()
if p == 10:
level1(score_stat)
elif p == 20:
level2(score_stat)
elif p == 30:
level3(score_stat)
elif p == 40:
level4(score_stat)
else:
end(score_stat)
def main():
#Loading the music for the title screen of the game and playing it infintely.
sound = pygame.mixer.Sound('C:\\Users\\Joseph Molina\\Desktop\\CST\\KeyGen.wav')
sound.play(loops = -1)
#Boolean to determine which sound file to play
start_music = True
#Setting the default value of score
total_score = 0
#using time variable as a way to keep track of the seconds
time = 0
#Loading the image of the background
backgroundImg = pygame.image.load('C:\\Users\\Joseph Molina\\Desktop\\CST\\space.jpg')
while True:
time = time + 1
#If statement to change the background image of the program
if time == 300:
backgroundImg = pygame.image.load('C:\\Users\\Joseph Molina\\Desktop\\CST\\anything.jpg')
if time == 500:
backgroundImg = pygame.image.load('C:\\Users\\Joseph Molina\\Desktop\\CST\\space.jpg')
if time == 700:
backgroundImg = pygame.image.load('C:\\Users\\Joseph Molina\\Desktop\\CST\\anything3.jpg')
if time == 900:
backgroundImg = pygame.image.load('C:\\Users\\Joseph Molina\\Desktop\\CST\\magicEarth.jpg')
if time == 990:
backgroundImg = pygame.image.load('C:\\Users\\Joseph Molina\\Desktop\\CST\\space.jpg')
global total_frames
current_score = BaseClass.total_score
process(bug, FPS, total_frames, SCREENHEIGHT, current_score)
#LOGIC
bug.motion(SCREENWIDTH, SCREENHEIGHT)
Enemy.update_all(SCREENWIDTH)
#Enemy.movement(SCREENWIDTH)
BugProjectile.movement()
total_frames += 1
#Bliting the image of the background into the screen at the given coordinates
screen.blit(backgroundImg, (0,0))
#Draws all the sprites to the screen
BaseClass.allsprites.draw(screen)
#If statement to change the pace of the music
score(BaseClass.total_score)
if BaseClass.total_score > 300 and start_music:
start_music = False
sound.stop()
GameMusic()
#Makes sure that everything is being drawn on the screen
pygame.display.flip()
#How many frames are going to be in a second
clock.tick(FPS)
def test_python_processing(self):
self.logger.debug("test python processing")
dataset = DataSet.objects.get(name=test_utils.TEST_DATA_SET)
pattern_list = PatternList.objects.get(name=test_utils.TEST_PATTERN_LIST)
process.process(dataset, pattern_list)
results = ResultSet.objects.all()
self.logger.info("Result sets: " + str(results.count()))
for r in results:
self.logger.debug(r)
pi = ProcessingInstance.objects.all()
self.assertEqual(len(pi), 1)
self.assertTrue(pi[0].successful, "test_python_processing")
def on_process(self, event_):
if self.validate():
timeid = time.strftime('%a/%H%M%S', time.localtime())
infile = self.filename
group_name = self.group_name.GetValue()
num_copies = self.num_copies.GetValue()
if num_copies != '':
process.process(infile, group_name, timeid, int(num_copies))
else:
process.process(infile, group_name, timeid)
shutil.copyfile(self.filename,
'outfiles/{}_{}.jpg'.format(timeid, tools.safe_filename(group_name)))
def on_process(self, event_):
if self.validate():
timeid = time.strftime('%a/%H%M%S', time.localtime())
infile = self.names[self.index]
group_name = self.group_name.GetValue()
num_copies = self.num_copies.GetValue()
if num_copies != '':
process.process('infiles/' + infile, group_name, timeid, int(num_copies))
else:
process.process('infiles/' + infile, group_name, timeid)
os.rename('infiles/' + infile,
'outfiles/{}_{}.jpg'.format(timeid, tools.safe_filename(group_name)))
self.load_next_image()
def handlein(filein, fileout):
lines = filein.readlines()
nb_books, nb_libs, days = map(int,lines[0].split(' '))
books = {}
book_scores = list(map(int,lines[1].split(' ')))
for i in range(nb_books):
books[i] = Book(i, book_scores[i], 0)
libs = []
lib_id = 0
for i in range(2,2+2*nb_libs,2):
capacity, signup, ship = map(int, lines[i].split(' '))
libbooks = list(map(lambda x: books[int(x)], lines[i +1].split(' ')))
libs.append(Library(lib_id,signup, ship, libbooks))
lib_id = lib_id + 1
score = 0
explored_libs=set()
explored_book =set()
print('len(explored_libs) :', len(explored_libs))
while(score < 4) :
explored_books=0
score =0
explored_libs.clear()
explored_book.clear()
print('Starting NbrLib Expl',len(explored_libs))
print('Starting score :', score)
print('Starting Explored ',explored_books)
for l in libs:
l.score = 0
l.scanned_book=0
l.booktokeep.clear()
l.max_score=0
l.sinupDate=0
for b in l.unique_books:
b.is_scanned=False
process(books, libs, days,explored_libs,explored_book)
print('NbrLib ',len(libs))
print('NbrBook ',len(books))
for b in explored_book :
score += books[b].score
explored_books +=1
print('Explored ',explored_books)
print('NbrLib Expl',len(explored_libs))
print("score: ",score)
generate_output(fileout, explored_libs)
def updateDatabase(range_, database):
ONE_DAY = datetime.timedelta(days = 1)
AREAS = ['AVIATION', 'CENTRAL EAST', 'CENTRAL WEST', 'DOWNTOWN',
'NORTH CENTRAL', 'NORTH EAST', 'NORTH WEST', 'OUT OF AREA',
'SOUTH CENTRAL', 'SOUTH EAST', 'SOUTH WEST']
for area in AREAS:
begin = range_[0]
end = range_[1]
while begin <= end:
process.process(os.path.join(settings.PATH_TO_DATA, area, str(begin)),
database)
begin += ONE_DAY
def main(config_path):
if config_path == None:
config_path = 'config/config.yml'
parseConfig(config_path)
global DOWNLOAD_PATH
DOWNLOAD_PATH = os.path.join(basePath, config["download"]["folder"] + "/")
PROCESSED_PATH = os.path.join(basePath,
config["processed"]["folder"] + "/")
list = s3.list_objects(
Bucket=config["s3"]["buckets"]["enriched"]["good"]["bucket"],
Prefix=(config["s3"]["buckets"]["enriched"]["good"]["path"] +
"/"))['Contents']
downloadFiles(list)
process(in_dir=DOWNLOAD_PATH, out_dir=PROCESSED_PATH)
def main():
args = docopt(
__doc__,
version="cyril version %s" % __VERSION__,
options_first=True,
)
for file in args['<file>']:
process(
file,
process_all=args['--all'],
remove_previous=args['--clean']
)
def main():
covids = listdir(data_covid)
for covid in covids:
folder = get_fname(covid)
for dname in data_dnames:
rname = f'{data_result}/{folder}/{dname.split("/")[-1]}'
mkdir(rname)
for fname in os.listdir(dname):
data = f'{dname}/{fname}'
oname = f'{rname}/{get_fname(fname)}.tif'
process.process(f'{data_covid}/{covid}', f'{dname}/{fname}',
oname)
def run(self):
temp_array = np.zeros(SHAPE, dtype='uint8')
while True:
start = time()
TEMP_COUNTER = self.COUNTER.value
if TAKEN_COUNTER.value < TEMP_COUNTER:
with TAKEN_COUNTER.get_lock():
TAKEN_COUNTER.value = TEMP_COUNTER
else:
sleep(0.0001 * randint(8, 15) * 2)
continue
np.copyto(temp_array, self.frame_pointer)
process(temp_array)
def process_data(filename):
try:
r = redis.Redis(connection_pool=redis_pool)
key = 'osrc::data::processed::{}'.format(filename)
if r.get(key) == 'True':
print("File {0} already processed".format(filename))
return
print("Start processing file: {0}".format(filename))
process(os.path.join(data_dir, filename))
print("Processed file: {0}".format(filename))
r.set(key, True)
except Exception as e:
print("Unable to process {0}: {1}".format(filename, e))
def main():
form_defaults = dict(request.form)
# Handle file save
# FILE_FIELDS = [k for k, v in FORM_SPECIFICATION.items() \
# if v[1] is FormInputs.FILE]
# for field_name in FILE_FIELDS:
# if field_name in request.files \
# and (file := request.files[field_name]).filename:
# # Save the file
# filename = secure_filename(file.filename)
# save_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
# file.save(save_path)
# # Update the form values
# form_defaults.update({field_name: save_path})
msg = None
if any(form_defaults.values()):
msg = process.process(**form_defaults)
form = utils.Form(FORM_SPECIFICATION, defaults=form_defaults)
return render_template('index.html',
form=form,
desc=FORM_DESCRIPTION,
msg=msg)
def on_process(self, _event):
if self.validate():
day = tools.get_day()
timeid = time.strftime('%H%M%S', time.localtime())
infile = self.names[self.index]
infile_path = os.path.join('infiles', infile)
group_name = self.group_name.GetValue()
num_copies = self.num_copies.GetValue()
if num_copies != '':
process.process(infile_path, group_name, day, timeid, int(num_copies)) # pylint: disable=too-many-function-args
else:
process.process(infile_path, group_name, day, timeid)
outfile_name = '{}_{}.jpg'.format(timeid, tools.safe_filename(group_name))
outfile_path = os.path.join('outfiles', day, outfile_name)
os.rename(infile_path, outfile_path)
self.load_next_image()
def handle_key_press (key):
global proc
if (key==SPACE):
lb.crossfader['AB'].get_fader('A').set_cue('1')
lb.crossfader['AB'].get_fader('A').set_level('100%')
lb.crossfader['AB'].get_fader('B').set_cue('2')
lb.crossfader['AB'].get_fader('B').set_level('0%')
lb.crossfader['AB'].run()
if (key>='1' and key<='9'):
lb.foo_program_name='prog'+str(key)
lb.program[lb.foo_program_name].run()
if (key=='?'):
print lb.program.keys()
if (key=='o'):
print "Which step?"
stps=map(lambda x: x.name, lb.program[lb.foo_program_name].actions)
print stps
q=string.strip(sys.stdin.readline())
print q
lb.program[lb.foo_program_name].set_next_step(step=stps.index(q))
#lb.program[lb.foo_program_name].next_step=q
#lb.program[lb.foo_program_name].step_forward()
if (key=='g'):
lb.program[lb.foo_program_name].step_forward()
if (key=='p'):
for e in lb.events:
print e
lb.events=[]
if (key=='c'):
proc=process.process()
proc.start(lb.procedure['circle'], instrument='moving1', center=(10, 10), radius=5)
if (key=='a'):
proc.stop()
if (key=='q'):
lb.exit()
def data_process(self): #get connectome, tau-list, and inverted connectome matrix
print(self.filename)
print(self.neural_list_filename)
self.invConnectome, self.tau_list, self.neuronCount = process.process(self.filename,self.neural_list_filename, int(self.E1.get()),int(self.E2.get()),float(self.E3.get()),int(self.E4.get()))
print('Inverted Connectome')
plt.spy(self.invConnectome)
plt.show(block=False)
def run(self):
target_ext = os.path.splitext(os.path.basename(
self.src_file_fullpath))[1]
if self.output_format in (
"JPEG",
"PNG",
"TIFF",
"BMP",
):
target_ext = "." + str(self.output_format).lower()
output_file = build_output_filename(self.src_file_fullpath,
self.output_dir, target_ext,
self.overwrite_files)
if self.hasCanceled:
self.hasFinished = True
self.emit(QtCore.SIGNAL("dofinished"), self.process_total,
self.src_file_fullpath)
return
ret = process(src_file = self.src_file_fullpath, dest_file = output_file,\
width = self.width, height = self.height,\
mode = self.mode, keep_datetime = self.keep_datetime)
self.hasFinished = True
self.emit(QtCore.SIGNAL("dofinished"), self.process_total,
self.src_file_fullpath)
def run():
defaultmech = "%s/mapping/cb05cl_ae6_aq.csv" % os.path.dirname(__file__)
parser = ArgumentParser(description = "Usage: %prog [-tq] \n"+(" "*16)+" [-i <init name>] [-f <final name>] <yamlfile>")
parser.add_argument("-t", "--template", dest = "template", action = "store_true", default = False, help="Output template on standard out (configurable with -m and -c")
parser.add_argument("-v", "--verbose", dest = "verbose", action = "count", default = 0, help = "extra output for debugging")
paths = glob(os.path.join(os.path.dirname(__file__), 'mapping', '*_*.csv'))
mechanisms = ', '.join(['_'.join(path.split('/')[-1].split('_')[:])[:-4] for path in paths])
parser.add_argument("-c", "--configuration", dest="configuration", default = None,
help = "Chemical mechanisms: %s (for use with -t)" % mechanisms)
parser.add_argument('configfile')
options = parser.parse_args()
args = [options.configfile]
if options.template:
from template import template
if options.configuration is None:
warn("Using default mechanism: %s" % defaultmech)
options.configuration = defaultmech
else:
if os.path.exists(options.configuration):
pass
else:
options.configuration = "%s/mapping/%s.csv" % (os.path.dirname(__file__), options.configuration)
if not os.path.exists(options.configuration):
raise IOError('Cannot find file %s; must be either you own file or in %s' % (options.configuration, mechanisms))
print template(options.configuration)
parser.exit()
if len(args)<1:
parser.error(msg="Requires a yaml file as an argument. For a template use the -t option. The template will be output to the stdout.")
else:
yamlpath=args[0]
from load import loader
from process import process
outf = process(config = loader(yamlpath), verbose = options.verbose)
def slug(path, slug):
L.info("dbserver::slug::%s:%s, %s, %s" %
(request.method, path, slug, request.url))
response, content_type, headers = process(path, slug)
return Response(response=response,
content_type=content_type,
headers=headers)
def handle_face_recognize(req):
rsp = face_recognizeResponse()
bridge = CvBridge()
ro_msg = UInt8MultiArray()
try:
for im in req.images_in:
cv_image = bridge.imgmsg_to_cv2(im, "bgr8")
# Leave image preprocess to client
result = ps.process(cv_image)
name_id_msg = UInt8
if result[1] > 0.9:
name_id_msg.data = result[0] + 1 # known face id start with 1
else:
name_id_msg.data = 0 # unknown face has id 0
ro_msg.data.append(name_id_msg)
# establish a dict to find the name refereed by the id
rsp.face_label_ids = ro_msg
return rsp
except CvBridgeError as e:
print(e)
return rsp
def scan():
json = flask.request.get_json()
if json is None:
flask.abort(400)
shared = json['shared']
if shared not in APILOCK:
APILOCK[shared] = manager.Lock() # pylint: disable=no-member
APIINIT[shared] = manager.Value('B', json['inited'])
info = Info(
name=json['name'],
uuid=json['uuid'],
mime=json['mime'],
report=json['report'],
inited=APIINIT[shared],
locked=APILOCK[shared],
workdir=json['workdir'],
environ=json['environ'],
install=json['install'],
scripts=json['scripts'],
)
RUNNING.append(info.uuid)
flag = process(info)
with contextlib.suppress(ValueError):
RUNNING.remove(info.uuid)
SCANNED[info.uuid] = flag
return flask.jsonify(id=info.uuid,
inited=info.inited.value,
scanned=True,
reported=flag,
deleted=False)
def accuracy_check(executor_type, test_parameters, output_handler, log):
process_executor = executor.get_executor(executor_type, log)
tests = test_parameters.get_config_data()
test_process = process(log, process_executor, test_parameters)
test_process.execute()
output_handler.add_results(test_process, tests)
log.info('Saving test result in file')
def search(search_game,feature_file):
result = None
sc2 = sc2reader.SC2Reader()
sg = sc2.load_replay(search_game)
search_data = process(sg)
if len(search_data) != 2:
return None
[(race1,v1),(race2,v2)] = search_data
best = float('inf')
for f,data in shelve.open(feature_file).iteritems():
p1_dist = playerGameDistance(race1,v1,data)
p2_dist = playerGameDistance(race2,v2,data)
if p1_dist == None or p2_dist == None:
continue
#d = p1_dist + p2_dist
d = p2_dist
if d < best:
best = d
result = f
print "Best: %s --" % result,best
print "Wooot!",result
def home(filename=None, bullets=None, captions=None):
if request.method == 'POST':
# check if the post request has the file part
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
file = request.files['file']
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
flash('No selected file')
return redirect(request.url)
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
basedir = os.path.abspath(os.path.dirname(__file__))
full_filename = os.path.join(app.config['UPLOAD_FOLDER'], filename)
file.save(full_filename)
overall_desc = process(full_filename)
bullets = overall_desc['bullets']
captions = overall_desc['image_descs']
# bullets = ['hi', 'this is a bullet', 'here is another bullet']
# captions = ['hi', 'this is a caption', 'this is another caption']
return render_template('index.html', filename=full_filename,
bullets=bullets, captions=captions)
elif request.method == 'GET':
return render_template('index.html', filename=filename, bullets=bullets, captions=captions)
def noslug(path):
L.info("dbserver::noslug::%s:%s, %s" % (request.method, path, request.url))
response, content_type, headers = process(path, None)
L.info("%s %s %s" % (snip(response), content_type, headers))
return Response(response=response,
content_type=content_type,
headers=headers)
def train_from_scratch(saving_loc, loss=None, initial_lr=0.0025, gpus=1):
n = 544
proc = process.process(loss,
raw,
gt,
aff,
model_type="heavy paralell UNET",
precision="half",
save_loc=saving_loc,
saving_sched=[[0, 100], [1000, 10000],
[100000, 20000]],
image_loc="tiffs/",
check_interval=100,
conf_coordinates=[[200, 216], [200, 328],
[200, 328]],
learning_rate=initial_lr,
validation_frac=.2,
validation_interval=200,
gpus=gpus)
flag = proc.train(500000)
while (not flag):
proc.iteration = 0
proc.learning_rate = proc.learning_rate * .1
print("\n\tNEW LR = %f" % proc.learning_rate)
flag = proc.train(500000)
def __init__(self,useLogger=True):
self.f = None
self.writer = writer.writer()
self.P = process.process(makeDirs=False)
self.gU= graphUtils.graphUtils()
self.pip= pipath.pipath()
self.requestedRuns = 'ALL'
self.now = now = datetime.datetime.now()
self.start_time = now
fmt = '%Y%m%d_%H%M%S_%f'
cnow = now.strftime(fmt)
self.parentDir = 'Second/'+cnow+'/'
self.logdir = self.parentDir + 'Log/'
self.figdir = self.parentDir + 'Figures/'
dirs = self.pip.fix( [self.parentDir, self.logdir, self.figdir] ) # make paths platform indepedent
for d in dirs:
if os.path.isdir(d):
pass
else:
try:
os.mkdir(d)
except IOError,e:
print 'second__init__',e
else:
print 'second__init__ created',d
def one_loop(args):
global parameters
import sys
sys.path.append(parameters["base_dir"])
from process import process
try:
res = process(args, parameters)
except Exception:
# get the traceback
tb = traceback.format_exc()
report = {
"args": args,
"tb": tb,
}
pid = os.getpid()
# now write the problem to file
fn_err = os.path.join(parameters["_results_dir"],
"error_{}.json".format(pid))
with open(fn_err, "a") as f:
f.write(json.dumps(report, indent=4))
f.write(",\n")
res = []
return res
def addProcess(self,
id,
title="",
description="",
BeginEnd=None,
priority=0,
limit=0,
valid_from=0,
valid_to=0,
waiting_time=0,
duration=0,
duration_unit='days',
REQUEST=None):
""" adds a new process """
p = process(id,
title,
description,
BeginEnd,
priority,
limit,
valid_from,
valid_to,
waiting_time,
duration,
duration_unit)
self._setObject(id, p)
if REQUEST: REQUEST.RESPONSE.redirect('Processes')
def main_with_args():
file_list = parse_args(sys.argv[1:])
if CPU_CNT <= 1:
[process(file) for file in sorted(file_list)] # pylint: disable=expression-not-assigned
else:
multiprocessing.Pool(CPU_CNT).map(process, sorted(file_list))
return 0
def quantization(executor_type, quantization_parameters, log):
process_executor = executor.get_executor(executor_type, log)
for i, params in enumerate(quantization_parameters):
log.info('Start quantization model #{}!'.format(i + 1))
quant_process = process(params, process_executor, log)
quant_process.execute()
log.info('End quantization model #{}!'.format(i + 1))
def process_play(raw_play_lines, gender, play_stats):
speaking_characters, play_lines = preprocess(raw_play_lines, matcher)
gender_stats(speaking_characters, gender, play_stats)
adj, act_scene_start_end = process(speaking_characters, play_lines)
play_stats['scenes'] = len(act_scene_start_end)
graph = postprocess(play_lines, speaking_characters, adj, gender,
act_scene_start_end, play_stats)
return play_lines, graph
def load(metas):
results = []
for meta in metas:
print u'解析目录: ', meta._dirname
for xlsx_file, xlsx_config in meta.iter_xlsx():
print u'解析Excel文件:', xlsx_file
for item_model, items in load_data(meta, xlsx_file, xlsx_config):
objs = []
for obj in items:
obj, errors, warnings = process(meta, obj, item_model)
obj['__errors'] = errors
obj['__warnings'] = warnings
obj, errors, warnings = process_ext(meta, obj, item_model)
obj['__errors'].extend(errors)
obj['__warnings'].extend(warnings)
objs.append(obj)
results.append([item_model, objs])
error_count = reduce(
operator.add,
[len(obj['__errors'])
for obj in itertools.chain(*[objs for item_model, objs in results])])
warning_count = reduce(
operator.add,
[len(obj['__warnings'])
for obj in itertools.chain(*[objs for item_model, objs in results])])
error_result_count = reduce(
operator.add,
[1
for obj in itertools.chain(*[objs for item_model, objs in results])
if len(obj['__errors']) > 0], 0)
result_count = reduce(operator.add, [len(objs)
for item_model, objs in results])
def print_summary():
print
print "=" * 40
print ' ' * 15, u'Summary'
print "=" * 40
print
print "数据量:", result_count
print "包含错误的数据量:", error_result_count
print "错误:", error_count
print "警告:", warning_count
if error_count > 0:
# 有错误，需先纠错
for obj in itertools.chain(*[objs for item_model, objs in results]):
if len(obj['__errors']) > 0:
print '-' * 40
print obj['__src'], obj['__src_sheet'], obj['__src_sheet_row']
print '\n'.join(obj['__errors'])
print_summary()
return results
def main():
# read parameters
parser = argparse.ArgumentParser(prog='Dake recommendation parser')
log_parameter = parser.add_argument_group('Log parameters', 'Parameters are used for logging.')
log_parameter.add_argument('--log_folder_path', default='logs', help='The log output position.')
log_parameter.add_argument('--log_level', default='DEBUG',
choices=['DEBUG', 'INFO', 'ERROR', 'WARNING', 'CRITICAL'], help='The log level.')
group_call = parser.add_argument_group('Group call', 'Parameters are used for calling this plugin.')
group_call.add_argument('--hive_cmd_env', default='XXX/hive',
help='The hive path which will be to used to run.')
args = parser.parse_args()
set_logging(args.log_folder_path, args.log_level)
logger = logging.getLogger()
logger.info('Begin program')
try:
process(args.hive_cmd_env)
except Exception, e:
logger.error(e.message)
raise
def parse_options(argv):
global func_dict, process_chain, mut_ex_func
func_dict = {
'-s' : lambda a : process(proc_shorten, a, 3, "-s", "Shorten url using bit.ly"),
'-t' : lambda a : process(proc_tar, a, 2, "-t", "Tar and gzip input before sending"),
'-z' : lambda a : process(proc_zip, a, 2, "-z", "Zip input before sending, takes precedence over -t"),
'-p' : lambda a : process(proc_pbin, a, 2, "-p", "Put input on pastebin, takes precedence over -t and -z"),
'-T' : lambda a : process(proc_test, a, 1, "-T", "Perform a test run"),
'-r' : lambda a : process(proc_recur, a, 1, "-r or -R", "Handle directories recursive"),
'-R' : lambda a : process(proc_recur, a, 1, "-r or -R", "Handle directories recursive"),
'-c:': lambda a : process(proc_conf, a, 1, "-c <file>", "Use <file> as configfile"),
'-v' : lambda a : process(proc_verb, a, 0, "-v", "Increase verbosity"),
'-h' : lambda a : process(proc_help, a, 0, "-h", "Show this help"),
}
functions = lambda o, a : (func_dict[o] if o in func_dict else func_dict[o + ":"])(a)
available_cli_options = reduce(lambda x, y: x + y, func_dict.keys())
try:
optlist, args = getopt.gnu_getopt(argv[1:], available_cli_options)
except getopt.GetoptError as oe:
log("Error, option \"-" + oe.opt + "\" is not available!\n")
usage(exit, 0)
for o, a in optlist:
process_chain.append(functions(o, a))
if(len(args) > 0):
for arg in args:
objects.append(pub_object(arg))
#we now know which process to run, they are in process_chain, but they are unsorted
#and may contain mutual exclusive options. before we can do anything, we need to
#remove the mutually exclusive options and sort them. first we make a list of these
#options, the first in the tuple takes precedence over the second
mut_ex_func = [
(proc_zip, proc_tar), #if you zip, you can't tar
(proc_pbin, proc_zip), #if you pastebin, you can't zip
(proc_pbin, proc_tar) #or tar
]
#then we filter out all mutex functions
fix_chain()
def generate(data_dir):
sc2 = sc2reader.SC2Reader()
d = shelve.open("features")
for f in os.listdir(data_dir):
if(len(f) > 9 and f[-9:].lower() == "sc2replay"):
print "Generating",f
replay = sc2.load_replay(os.path.join(data_dir,f))
result = process(replay)
d[f] = result
d.close()
def __init__(self):
"""
Here we'll load settings and set up modules.
"""
self.barf = barf.Barf
self.process = process.process()
self.settings = '' # compat, ugly.
self.debug = self.getsetting('brain', 'debug')
self.barf('MSG', 'Scrib %s initialized' % self.process.version)
def main():
#mystr = "изучением видных советских теоретиков права, которые были приведены выше."
print "Content-type:text/html\n"
form = cgi.FieldStorage()
mystr = form.getvalue('name')
#t = threading.Thread(target = printok, args = (15,))
#t.daemon = True
#t.start()
expert_list, categ_list=process(mystr)
#print expert_list, categ_dict
print json_formation(expert_list, categ_list)#['http://www.hse.ru/org/persons/201866', 'http://www.hse.ru/org/persons/140394']
def process_loop(self, apps, helper):
children = []
for app in apps:
self.curlInstance.limit_threads(children)
p = process.process(self.config, self.curlInstance, app, self.config.get_section_items(app))
child = threading.Thread(target=helper, args=[p])
children.append(child)
child.start()
self.curlInstance.clear_threads(children)
def post(self, request, *args, **kwargs):
pattern_list_id = int(request.POST['pattern_lists'])
dataset_id = int(request.POST['dataset_id'])
dataset = DataSet.objects.get(id=dataset_id)
pattern_list = PatternList.objects.get(id=pattern_list_id)
context = super(DatasetProcessView, self).get_context_data(**kwargs)
context['DATASET'] = dataset
context['PATTERN_LIST'] = pattern_list
process_instance = process(dataset, pattern_list)
context['PROCESS_INST_ID'] = process_instance.id
return self.render_to_response(context)
def get_downloaded_filename(self, app_name):
p = process.process(self.configuration, self.curl_instance, app_name, self.apps[app_name])
filename = p.replace_version_with_mask(self.apps[app_name][process.APP_FILENAME])
try: rename = p.replace_version_with_mask(self.apps[app_name][process.APP_RENAME])
except KeyError: rename = ''
filemask = self.curl_instance.get_cached_name(filename, rename)
file = self.get_first_from_mask(filemask)
if file == '':
print 'File not downloaded: %s' % filemask
sys.exit(1)
print '- Downloaded: %s' % file
return file
def tick(self):
process(self.app, self.player)
self.player.motion(self.app.resolution)
if self.player.alive():
if randint(1, self.difficulty) == 1:
if randint(1, 8) == 3:
Enemy(randint(0, 320), -40, 34, 34, 'resources/rock_large.png', self.player, 500)
else:
Enemy(randint(0, 320), -30, 18, 18, 'resources/rock.png', self.player, 200)
if self.difficulty > 10 and randint(1, 8) == 1:
self.difficulty -= 1
for enemy in Enemy.List:
enemy.motion(self.app.resolution, self.difficulty)
for explosion in Explosion.List:
explosion.update()
self.score = self.app.font.render('score: ' + str(self.player.score), True, (255, 255, 0))
self.fpst = self.app.font.render('fps: ' + str(int(self.app.clock.get_fps())), True, (255, 255, 0))
else:
self.app.running = False
self.draw()
def run(self):
target_ext = os.path.splitext(os.path.basename(self.src_file_fullpath))[1]
if self.output_format in ("JPEG", "PNG", "TIFF", "BMP",):
target_ext = "." + str(self.output_format).lower()
output_file = build_output_filename(self.src_file_fullpath, self.output_dir, target_ext, self.overwrite_files)
if self.hasCanceled:
self.hasFinished = True
self.emit(QtCore.SIGNAL("dofinished"), self.process_total, self.src_file_fullpath)
return
ret = process(src_file = self.src_file_fullpath, dest_file = output_file,\
width = self.width, height = self.height,\
mode = self.mode, keep_datetime = self.keep_datetime)
self.hasFinished = True
self.emit(QtCore.SIGNAL("dofinished"), self.process_total, self.src_file_fullpath)
def runFile():
f=open(sys.argv[1])
statement=''
for buff in f.xreadlines():
statement+=buff.split(';')[0]
statement=statement.strip()
if not statement or not parentheseBalance(statement):
continue
if statement:
try:
val=process(parse(statement),global_env)
display(val)
except IOError as err:
print "[error]%s"%(err,)
statement=''
def __init__(self):
self.useragent = "rDany/1.1 (http://www.rdany.org/rdany/; [email protected])"
self.processors = process(self.useragent)
self.telegram_conection = telegram(Config.bot_username, Config.bot_token, self.useragent)
self.end = False
self.context = {
"general.last_processor": "",
"general.last_confidence": 0,
"general.last_time": 0,
"general.last_search": "",
"general.total_questions": 0,
"general.succesful_answers": 0,
"general.processors_examples": self.processors.get_examples(),
"shared.verbosity": False
}
