def download_massey_csvs(self):
if not os.path.exists(self.data_dir):
if os.name is 'nt':
os.makedirs(self.data_dir)
else:
os.makedir(self.data_dir)
for year in self.years:
if not os.path.isfile(self._game_filename(year)):
games = self._scrape_game_data(year)
try:
game_file = open(self._game_filename(year), "w")
print "Writing " + self._game_filename(year) + "..."
game_file.write(games.read())
game_file.close()
except:
print "Error: could not write file " + self._game_filename(year)
quit()
if not os.path.isfile(self._team_filename(year)):
teams = self._scrape_team_data(year)
try:
team_file = open(self._team_filename(year), "w")
print "Writing " + self._team_filename(year) + "..."
team_file.write(teams.read())
team_file.close()
except:
print "Error: could not write file " + self._team_filename(year)
quit()
def verify_dir(dir):
"""
Checks if a directory exists, if not, it creates it.
Args:
path (str): name of directory to check
Returns:
path object for directory
Raises:
OSError if unable to create the directory due to permissions
or whatnot.
RuntimeError if the dir is a file.
"""
path = Path(dir)
if os.path.isdir(path):
pass
elif os.path.isfile(dir):
raise RuntimeError("The dir is a file")
elif not os.path.exists(dir):
try:
os.makedir(path)
except OSError:
raise
return path
def POST(self, type):
file = web.input(avatar={}, background={})
file_dir = os.get_cwd(
) + "/static/uploads/" + session_data['user']['username']
if not os.path.exists(file_dir):
os.makedir(file_dir)
if "avatar" or "background" in file:
filepath = file[type].filename.replace('\\', '/')
filename = filepath.split('/')[-1]
f = open(file_dir + "/" + filename, 'wb')
f.write(file[type].file.read())
f.close()
update = {}
update['type'] = type
update['img'] = '/static/uploads' + session_data["user"][
"username"] + "/" + filename
update['username'] = session_data['user']['username']
account_model = LoginModel.LoginModel()
update_avatar = account_model.update_image(update)
raise web.seeother('/settings')
def Reload(self):
if len(self.PATH_PROJECT) > 0 and len(self.NAME_PROJECT) > 0:
Projectfolder = os.path.join(self.PATH_PROJECT, self.NAME_PROJECT)
Inputfolder = os.path.join(self.PATH_PROJECT, self.NAME_PROJECT,
'Input_Media_File')
OutputOpenMVG = os.path.join(self.PATH_PROJECT, self.NAME_PROJECT,
'OpenMVG_Output')
OutputOpenMVS = os.path.join(self.PATH_PROJECT, self.NAME_PROJECT,
'OpenMVS_Output')
OutputOpenMVSScene = os.path.join(self.PATH_PROJECT,
self.NAME_PROJECT,
'OpenMVS_Output', 'Scene')
if not os.mkdir(Projectfolder):
os.makedir(Projectfolder)
if not os.mkdir(Inputfolder):
os.makedir(Inputfolder)
if not os.mkdir(OutputOpenMVG):
os.makedir(OutputOpenMVG)
if os.mkdir(OutputOpenMVS):
os.makedir(OutputOpenMVS)
if os.mkdir(OutputOpenMVSScene):
os.makedir(OutputOpenMVSScene)
self.Enter_Text_From_File()
else:
self.ui.textEdit_Console.moveCursor(QTextCursor.End)
self.ui.textEdit_Console.insertPlainText(
'\n Project not Create or Select')
def touch(path):
basedir = os.path.dirname(path)
if not os.path.exists(basedir):
os.makedir(basedir)
with open(path, 'a'):
os.utime(path, None)
def generate_training_batch(start_case, end_case):
print("generating batch from %d to %d", start_case, end_case)
if not os.path.isdir(BATCH_DIR):
os.makedir(BATCH_DIR)
if not os.path.isdir(KEY_DIR):
os.makedir(KEY_DIR)
batch_name = str(start_case) + "-" + str(end_case)
batch_path = os.path.join(BATCH_DIR, batch_name)
key_path = os.path.join(KEY_DIR, batch_name)
if os.path.isfile(batch_path):
return
for case_id in range(start_case, end_case + 1):
# find how many slices for per case_id
pwi_dir = os.path.join(DATASET_DIR, "case_" + str(case_id), "PWI")
num_of_slices = len(os.listdir(pwi_dir))
print("case id - num of slices: ", case_id, num_of_slices)
if num_of_slices == 0:
return
for slice_id in range(num_of_slices):
batch, key = get_training_set_by_case(case_id, slice_id)
with open(batch_path, "w+") as f:
np.savetxt(f, batch, fmt="%.6f")
with open(key_path, "w+") as g:
np.savetxt(g, key, fmt="%d")
f.close()
g.close()
def __init__(self, handle):
""" Constructor
"""
# Call super class "Activity" constructor method
super(ScreencastActivity, self).__init__(handle)
# User interface object
self._ui = sc_ui.ScreencastUserInterface(self)
# Set video file
directory = os.path.join(get_activity_root(), "data")
if not os.path.exists(directory):
os.makedir(directory)
self._videofile = os.path.join(directory, "screencast.ogg")
# Gstreamer interface object
self._gst = sc_gst.ScreencastGstreamer(self._videofile)
# State
self._state = "stopped"
# Connect user interface signals
self._ui.connect('recordbutton-clicked', self.recordbuttonClicked)
self._ui.connect('stopbutton-clicked', self.stopbuttonClicked)
# Connect gstreamer interface signals
self._gst.connect('update-info', self.updateInfo)
# Show user interface
self._ui.showInterface()
def build(config):
global BUILD
BUILD = "release"
if not qprompt.ask_yesno("Build release (else debug)?", dft=config['release']):
BUILD = "debug"
# Build application.
call("uic mainwindow.ui >> ui_mainwindow.h")
call("qmake")
if not os.path.exists(BUILD):
os.makedir(BUILD)
status = call("make %s" % (BUILD))
try:
# Copy over Qt DLLs.
for dll in config['qt_dlls']:
src = os.path.join(config['qt_bin_dir'], dll)
shutil.copy(src, BUILD)
# Copy over assets.
for ast in config['assets']:
shutil.copy(ast, BUILD)
# Remove unnecessary files from build dir.
for f in os.listdir(BUILD):
if any([f.endswith(e) for e in config['build_dir_ext_rm']]):
os.remove(os.path.join(BUILD, f))
except:
qprompt.alert("File copy failed! Try killing app.")
status = 1
# Run app.
if status == 0:
if qprompt.ask_yesno("Run app?", dft=True):
run()
else:
qprompt.alert("ERROR: Build failed!")
def makedir_local(self,path): if not os.path.exists(path): # no need to make a dir that exists os.makedirs(path) elif os.path.isfile(path): # if there is a file there ditch it os.remove(path) del self.files[path] os.makedir(path)
def load_voice(url):
if os.path.exists('sound'):
urls = os.listdir('sound')
for temp_url in urls:
os.remove('sound/' + temp_url)
else:
os.makedir('sound')
f = wave.open(url)
params = f.getparams()
nchannels, sampwidth, framerate, nframes = params[:4]
sr = framerate
f.close()
shift = -9.5
y, sr = librosa.load(url, sr=sr)
for i in range(0, 21):
if i % 7 == 3 or i % 7 == 0:
shift += 0.5
else:
shift += 1
z = librosa.effects.pitch_shift(y, sr, shift)
#librosa.output.write_wav('sound/'+str(shift)+'.wav',y,sr=44100)
sf.write('sound/' + str(shift) + '.wav', z, sr)
sounds_url = os.listdir('sound')
sounds_url = sorted(sounds_url, key=lambda x: float(x[:-4]))
sounds = map(lambda x: pygame.mixer.Sound('sound/' + x), sounds_url)
key_sound = dict(zip(keys, sounds))
for key in key_sound:
key_sound[key].set_volume(0.05)
return key_sound
def Minimisation_Function(cluster, collection, cluster_name):
#######################################################################################
cluster.pbc = True # make sure that the periodic boundry conditions are set off
#######################################################################################
# Perform the local optimisation method on the cluster.
original_path = os.getcwd()
offspring_name = str(cluster_name)
clusters_to_make_name = 'clusters_for_VASP'
if not os.path.exists(clusters_to_make_name):
os.makedir(clusters_to_make_name)
copyfile('VASP_Files/INCAR',
clusters_to_make_name + '/' + offspring_name + '/INCAR')
copyfile('VASP_Files/POTCAR',
clusters_to_make_name + '/' + offspring_name + '/POTCAR')
copyfile('VASP_Files/KPOINTS',
clusters_to_make_name + '/' + offspring_name + '/KPOINTS')
os.chdir(clusters_to_make_name + '/' + offspring_name)
ase_write(cluster, 'POSCAR', 'vasp')
startTime = time.time()
try:
Popen(['srun', 'vasp'])
except Exception:
pass
endTime = time.time()
cluster = ase_read('OUTCAR')
os.chdir(original_path)
####################################################################################################################
# Write information about the algorithm
Info = {}
Info["INFO.txt"] = ''
#Info["INFO.txt"] += ("No of Force Calls: " + str(dyn.get_number_of_steps()) + '\n')
Info["INFO.txt"] += ("Time (s): " + str(endTime - startTime) + '\n')
#Info["INFO.txt"] += ("Cluster converged?: " + str(dyn.converged()) + '\n')
####################################################################################################################
return cluster, converged, Info
def main():
mgr = LBRYLinkManager()
urls = mgr.get_urls() # verbose=True
# print("URLs: {}".format(urls))
AppImage_URLs = []
for url in urls:
if url.endswith(".AppImage"):
if url not in AppImage_URLs:
AppImage_URLs.append(url)
print("AppImage_URLs: {}".format(AppImage_URLs))
if len(AppImage_URLs) > 1:
print("WARNING: There is more than one AppImage: {}"
"".format(AppImage_URLs))
dls_path = mgr.get_downloads_path()
if not os.path.isdir(dls_path):
os.makedir(dls_path)
for url in AppImage_URLs:
fname = url.split("/")[-1]
src_path = os.path.join(dls_path, fname)
if os.path.isfile(src_path):
print("* removing download not installed: {}"
"".format(src_path))
os.remove(src_path)
print("* downloading {}".format(url))
mgr.download(src_path, url, cb_progress=d_progress, cb_done=d_done)
print("* installing {}".format(src_path))
install_program_in_place(
src_path,
caption="LBRY",
move_what='file',
do_uninstall=False,
enable_reinstall=False
)
return 0
def make_folder(path):
'''
创建文件夹
'''
if exists_file(path):
return
os.makedir(path)
def train(self, model, train_data, valid_data, load_model=False):
if load_model == True:
model = load_model(self.model_path + 'tsc.h5')
(trainX, trainY) = train_data
(validX, validY) = valid_data
augmentor = ImageDataGenerator(rotation_range=10,
zoom_range=0.15,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.15,
horizontal_flip=False,
vertical_flip=False,
fill_mode='nearest')
history = model.fit_generator(
augmentor.flow(trainX, trainY, batch_size=self.BATCH_SIZE),
validation_data=(validX, validY),
steps_per_epoch=trainX.shape[0] // self.BATCH_SIZE,
epochs=self.NUM_EPOCHS,
class_weight=self.class_weight)
if not os.path.exists(self.model_path):
os.makedir(self.model_path)
model.save(self.model_path + 'tsc.h5')
return history, model
def check_arguments():
global DEPTH_KEY
global VAF_KEY
if not os.path.exists(options.vcfdir):
print("Invalid VCF folder %s"%(options.vcfdir))
return False
if not os.path.exists(options.outdir):
print("Creating output folder %s"%(options.outdir))
try:
os.makedir(options.outdir)
except OSError:
print("Invalid / unable to create, output folder %s"%(options.outdir))
return False
if options.format == "GATK":
DEPTH_KEY="AD"
VAF_KEY="AD"
if options.format == "FREEB":
DEPTH_KEY = "DP"
VAF_KEY = "DPR"
print("Running with the following settings:")
print("------------------------------------")
print(options)
print("DEPTH FIELD:"+DEPTH_KEY)
print("ALLELE FIELD:"+VAF_KEY)
print("------------------------------------")
return True
def pickle_datasets(df_train, df_dev, df_test):
if not os.path.exists(LYRICS_CNN_DIR):
os.makedir(LYRICS_CNN_DIR, exist_ok=True)
picklify(df_train, LYRICS_CNN_DF_TRAIN_PICKLE)
picklify(df_dev, LYRICS_CNN_DF_DEV_PICKLE)
picklify(df_test, LYRICS_CNN_DF_TEST_PICKLE)
return
def Init_Uploads():
path = os.getcwd()
UPLOAD_FOLDER = os.path.join(path, 'uploads')
PRIVATE_FOLDER = os.path.join(path, 'private')
if not os.path.isdir(UPLOAD_FOLDER) and os.path.isdir(PRIVATE_FOLDER):
os.makedir(UPLOAD_FOLDER)
os.makedir(PRIVATE_FOLDER)
def check_folder(folder):
""" checa se a pasta existe caso contrario cria """
if os.path.exists(folder) and os.path.isdir(folder):
return folder
else:
os.makedir(folder)
return folder
def save(self):
array_filename = self.get_array_filename()
directory = os.path.dirname(array_filename)
if not os.path.exists(directory):
os.makedir(directory)
self.depth.dump(self.get_array_filename)
logger.debug("saved depth data to %s", array_filename)
def openVideo(window_name, path_name, max_pic=100, camera_id=0):
if not os.path.exists("%s/" % path_name):
os.makedir("%s/" % path_name)
cv2.namedWindow(window_name)
capture = cv2.VideoCapture(camera_id)
num = 0
while capture.isOpened():
ok, img = capture.read()
if ok:
if detect(img, num, path_name):
num = num + 1
cv2.imshow(window_name, img)
keycode = cv2.waitKey(1)
if keycode == 27:
break
if num > max_pic:
break
capture.release()
cv2.destroyAllWindows()
def rewrite_tsv(file_path, rewrite_row, count_deltas=None):
"""Loop through the file at file_path and call
rewrite_row for each row. Modified rows are written
to a new output file.
"""
path, fname = os.path.split(file_path)
dirpath, dirname = os.path.split(path)
out_dirpath = os.path.join(dirpath, dirname+"_modified")
if not os.path.exists(out_dirpath):
os.makedir(out_dirpath)
with open(file_path, 'rb') as in_f:
tsvr = csv.reader(
in_f, delimiter='\t', encoding='utf-8', quotechar="|",
quoting=csv.QUOTE_NONE, lineterminator='\n'
)
with open(os.path.join(out_dirpath, fname), 'wb') as out_f:
tsvw = csv.writer(
out_f, delimiter='\t', encoding='utf-8', quotechar="|",
quoting=csv.QUOTE_NONE, lineterminator='\n', escapechar=None
)
for i, row in enumerate(tsvr):
mod_rows = rewrite_row(row, count_deltas)
for row in mod_rows:
try:
tsvw.writerow(row)
except csv.Error:
clean = clean_row(row)
tsvw.writerow(clean)
if i % 10000 == 0:
print "Processed %i0K rows from %s" % (i/10000, file_path)
return count_deltas, out_dirpath
def checkDir(filedir):
for dirname in (ROOTDIR, SPLITDIR, FILEDIR):
if not os.path.exists(dirname):
os.makedir(dirname)
for file in os.listdir(filedir):
if file.endswith(".png"):
os.remove(os.path.join(filedir, file))
def upload(user_id, image_id):
file = request.files['file']
filename = secure_filename(file.filename)
temp_path = os.path.join(app.config['UPLOAD_FOLDER'], user_id)
if not os.path.exists(temp_path):
os.makedir(temp_path)
file.save(temp_path, image_id+Path(filename).suffix)
def __init__(self):
'''
Instantiate the object with some values related to
the schema of the xls files
'''
if not os.path.isdir(DATA_DIR):
os.makedir(DATA_DIR)
self.outFile = os.path.join(DATA_DIR, 'raw_tax_data.tsv')
self.sheet_index = 0
# Specifying where the data is located in the sheet
self.min_row = 27
self.max_row = 20000
self.min_col = 1
self.max_col = 14
# Define the layout of the file
self.columns_map = {
0 : {"description" : u"DEP", "name" : u"dep"} ,
1 : {"description" : u"Commune", "name" : u"commune"} ,
2 : {"description" : u"Libellé de la communes", "name" : u"label_commune"} ,
3 : {"description" : u"Revenu fiscal de référence par tranche (en euros)", "name" : u"segment_fiscal_revenue"} ,
4 : {"description" : u"Nombre de foyers fiscaux", "name" : u"count_fiscal_hh"} ,
5 : {"description" : u"Revenu fiscal de référence des foyers fiscaux", "name" : u"ref_fiscal_rev_fiscal_hh"} ,
6 : {"description" : u"Impôt net (total)", "name" : u"net_tax"} ,
7 : {"description" : u"Nombre de foyers fiscaux imposables", "name" : u"count_fiscal_hh_taxable"} ,
8 : {"description" : u"Revenu fiscal de référence des foyers fiscaux imposables", "name" : u"ref_fiscal_rev_fiscal_hh_taxable"} ,
9 : {"description" : u"Traitements et salaires - Nombre de foyers concernés", "name" : u"salaries_actual_hh_count"} ,
10: {"description" : u"Traitements et salaires - Montant", "name" : u"salaries_amount"} ,
11: {"description" : u"Retraites et pensions - Nombre de foyers concernés", "name" : u"retirement_actual_hh_count"} ,
12: {"description" : u"Retraites et pensions - Montant", "name" : u"retirement_amount"}
}
def transformation_on_adv(args,
dataset,
defense_name,
adv_params,
data_batch_idx=None):
pool = ThreadPool(args.n_threads)
def generate(idx):
return dataset[idx]
dataset = pool.map(generate, range(len(dataset)))
pool.close()
pool.join()
# save all data in a file
all_adv = []
all_targets = []
for item in dataset:
all_adv.append(item[0])
all_targets.append(item[1])
all_adv = torch.stack(all_adv, 0)
all_targets = torch.LongTensor(all_targets)
if not os.path.isdir(args.partition_dir):
os.makedir(args.partition_dir)
start_idx, end_idx = _get_start_end_index(args)
out_file = paths.get_adversarial_file_path(args, args.partition_dir,
defense_name, adv_params,
end_idx, start_idx,
data_batch_idx)
torch.save({'all_outputs': all_adv, 'all_targets': all_targets}, out_file)
print('Saved Transformed tensor at ' + out_file)
dataset = None
all_adv = None
def download_images(img_urls, dest_dir):
"""Given the urls already in the correct order, downloads
each image into the given directory.
Gives the images local filenames img0, img1, and so on.
Creates an index.html in the directory
with an img tag to show each local image file.
Creates the directory if necessary.
"""
# +++your code here+++
if os.path.exists(dest_dir) == False:
os.makedir(dest_dir)
index = file(os.path.join(dest_dir, 'index.html'), 'w')
index.write('<verbatim>\n<html>\n<body>\n')
i = 0
for url in img_urls:
file_name = 'img' + str(i)
downloaded_file = os.path.join(dest_dir, file_name)
urllib.urlretrieve(url, downloaded_file)
index.write('<img src="' + str(downloaded_file) + '">')
i += 1
index.write('</body>\n</html>')
webbrowser.open(
'file:///Users/eric/desktop/google/exercises/logpuzzle/index.html')
def main(inputfile, start_frame=-1, end_frame=-1, move=False, output=None):
"""
Remove or move files from paths described in a pathfile.
This function will remove/delete all files in a pathfile
in case `start_frame` and `end_frame` equal to -1. In case
of moving file, an output folder may be passed as argument.
"""
#print inputfile, start_frame, end_frame, move, output
if move and not output:
output = join(dirname(inputfile), 'removed')
if not isdir(output):
legger.info('Creating folder: {}'.format(output))
os.makedir(output)
dicfiles = {}
pb = ProgressBar(count_lines(inputfile))
with open(inputfile) as fin:
for line in fin:
path = line.strip().split()[0]
fname, _ = splitext(basename(path))
fname = int(fname)
dicfiles[fname] = path
pb.update()
for id in sorted(dicfiles):
if (start_frame <= id and id <= end_frame) \
or (start_frame == -1 and end_frame == -1):
try:
if move:
shutil.move(dicfiles[id], join(output, str(id) + '.jpg'))
else:
logger.info('Removing file {}'.format(dicfiles[id]))
os.remove(dicfiles[id])
except:
logger.warning('File {} does not exist!'.format(dicfiles[id]))
def load_batch(data, img_ch, img_cols, img_rows):
# useful for debug
save_images = False
# if save images, create the related directory
img_dir = "images"
if save_images:
if not os.path.exists(img_dir):
os.makedir(img_dir)
# create batch structures
left_eye_batch = np.zeros(shape=(data[0].shape[0], img_ch, img_cols,
img_rows),
dtype=np.float32)
right_eye_batch = np.zeros(shape=(data[0].shape[0], img_ch, img_cols,
img_rows),
dtype=np.float32)
face_batch = np.zeros(shape=(data[0].shape[0], img_ch, img_cols, img_rows),
dtype=np.float32)
face_grid_batch = np.zeros(shape=(data[0].shape[0], 1, 25, 25),
dtype=np.float32)
y_batch = np.zeros((data[0].shape[0], 2), dtype=np.float32)
# load left eye
for i, img in enumerate(data[0]):
img = cv2.resize(img, (img_cols, img_rows))
if save_images:
cv2.imwrite(join(img_dir, "left" + str(i) + ".png"), img)
img = image_normalization(img)
left_eye_batch[i] = img.transpose(2, 0, 1)
# load right eye
for i, img in enumerate(data[1]):
img = cv2.resize(img, (img_cols, img_rows))
if save_images:
cv2.imwrite("images/right" + str(i) + ".png", img)
img = image_normalization(img)
right_eye_batch[i] = img.transpose(2, 0, 1)
# load faces
for i, img in enumerate(data[2]):
img = cv2.resize(img, (img_cols, img_rows))
if save_images:
cv2.imwrite("images/face" + str(i) + ".png", img)
img = image_normalization(img)
face_batch[i] = img.transpose(2, 0, 1)
# load grid faces
for i, img in enumerate(data[3]):
if save_images:
cv2.imwrite("images/grid" + str(i) + ".png", img)
face_grid_batch[i] = img.reshape((1, img.shape[0], img.shape[1]))
# load labels
for i, labels in enumerate(data[4]):
y_batch[i] = labels
return [right_eye_batch, left_eye_batch, face_batch,
face_grid_batch], y_batch
def go():
sources=['bt', 'city_lights', 'esm', 'ndvi', 'population_count',\
'population_density', 'soil_sealing', 'srtm', 'vk_heatmap']
years=[k for k in range(2000, 2018)]
X = get_dependent_variable(years=years)
y = get_independent_variable(years=years)
X1, y1 = augement_data(np.reshape(X, (1, X.shape[0], X.shape[1], X.shape[2], X.shape[3], X.shape[4])), y, 10)
train_X = np.reshape(X, (X.shape[0], X.shape[1] * X.shape[2], X.shape[3], X.shape[4]))
train_y = y
val_X = np.reshape(X1, (X1.shape[0], X1.shape[1] * X1.shape[2], X1.shape[3], X1.shape[4]))
val_y = y1
INIT_LR = 1e-3
BATCH_SIZE = 32
EPOCHS = 2
tbCallBack = keras.callbacks.TensorBoard(log_dir='./Graph2', histogram_freq=2, write_graph=True, write_images=True,
write_grads=True)
model = create_model(number_of_feat=len(sources))
model.compile(
loss='mse',
optimizer=keras.optimizers.adamax(lr=INIT_LR),
metrics=['mse']
)
model.fit(val_X, val_y, batch_size=BATCH_SIZE,
epochs=EPOCHS, callbacks=[tbCallBack], validation_data=(train_X, train_y))
if not os.path.exists("trained_models/"):
os.makedir("trained_models/")
model_path = "trained_models/model.hdf5"
print("Saving model weights to " + model_path)
model.save_weights(model_path)
def write_default_config():
"""
Check to see if there is a preexisting config file and if there isn't make
a new one with the proper defaults
"""
if os.path.exists(os.path.expanduser("~/.config/vboxoverlord/vbo.conf")):
return
else:
if not os.path.exists(os.path.expanduser("~/.config/vboxoverlord/")):
os.makedir(os.path.expanduser("~/.config/vboxoverlord"))
config = {
"global": {
"username": "******",
"port": 22,
},
"servers": {
"local": "localhost",
},
"init_levels": {
"1": "",
"2": "",
"3": "",
"4": "",
"5": "",
},
}
write_config(config)
def save(self):
array_filename = self.get_array_filename()
directory = os.path.dirname(array_filename)
if not os.path.exists(directory):
os.makedir(directory)
self.depth.dump(self.get_array_filename)
logger.debug("saved depth data to %s", array_filename)
def uploadHandler(msg, hashTag):
#print(msg)
file_id = msg['photo'][-1]['file_id']
userName = msg['chat']['first_name']
"""Convert the integer data type to string"""
date = str(msg['date'])
#print(userName, file_id, date)
"""
Use the download_file function from telepot module to download.
Save all the photo(s) in 'Files' directory located under root directory
"""
#print(os.getcwd())
if 'Files' in os.getcwd():
telegram_config.bot.download_file(file_id, userName + "_" + hashTag + "_" + date + ".jpg")
elif 'Codes' in os.getcwd():
"""Check Files directory exist"""
if not (os.path.exists('../Files/')):
"""Create new directory called Files"""
os.makedir('../Files/')
os.chdir('../Files/')
#print(os.getcwd())
telegram_config.bot.download_file(file_id, '' + userName + "_" + hashTag + "_" + date + ".jpg")
else:
if not (os.path.exists('./Files/')):
os.makedirs('./Files/')
os.chdir('./Files')
#print(os.getcwd())
telegram_config.bot.download_file(file_id, userName + "_" + hashTag + "_" + date + ".jpg")
## End of uploadHandler
def main(ch, m, savedir, opts, n=10000):
if not os.path.exists(savedir):
os.makedir(savedir)
scramble = parse_bool(opts[1])
xss = np.logspace(-2, 2, 30)
results = np.recarray((len(xss) * n),
dtype=[('bg_ts', float), ('sig_ts', float),
('inj_xs', float), ('fit_xs', float)])
for i, xs in enumerate(xss):
mu_bg, mu_s, bg_tmpl, sig_tmpl = get_distributions(ch, m, xs, scramble)
slc = slice(i * n, (i + 1) * n)
null_TS = np.zeros(n)
signal_TS = np.zeros(n)
fit_xs = np.zeros(n)
for i in range(n):
lxs_ini = np.log(10) * (np.log10(xs) + 0.5 *
(np.random.rand() - 0.5))
null_data = np.random.poisson(mu_bg)
ts = TS(null_data, sig_tmpl, bg_tmpl, lxs_ini=lxs_ini)
null_TS[i] = ts.get_TS()
inj_data = np.random.poisson(mu_bg + xs * mu_s)
ts = TS(inj_data, sig_tmpl, bg_tmpl, lxs_ini=lxs_ini)
signal_TS[i] = ts.get_TS()
fit_xs[i] = np.exp(ts.signal_fit.x[0]) * 1e-39
results['bg_ts'][slc] = null_TS
results['sig_ts'][slc] = signal_TS
results['inj_xs'][slc] = xs * 1e-39
results['fit_xs'][slc] = fit_xs
np.save('%s/ch%d-m%d_%s_realiztions' % (savedir, ch, m, opts), results)
def main(args):
if not os.path.exists(args.output_dir):
os.makedir(args.output_dir + '/' + args.name)
print("make dir:", args.output_dir + '/' + args.name + '/')
cam = cv2.VideoCapture(0)
print("Open Camera.")
num = 0
while cam.isOpened():
_, frame = cam.read()
cv2.putText(frame, "S:Save image.", (10, 300),
cv2.FONT_HERSHEY_COMPLEX, 0.8, (0, 0, 0), 2, cv2.LINE_AA)
cv2.putText(frame, "Q:Quite.", (10, 330), cv2.FONT_HERSHEY_COMPLEX,
0.8, (0, 0, 0), 2, cv2.LINE_AA)
flag = cv2.waitKey(1)
cv2.imshow("Photograph", frame)
# 按s拍照存储
# 按q退出程序
if flag == ord('s'):
filename = args.output_dir + '/' + args.name + '/' + args.name + str(
num) + '.jpg'
cv2.imwrite(filename, frame)
print("Save image: ", filename)
num += 1
elif flag == ord('q'):
break
cam.release()
cv2.destroyAllWindows()
def _log_complete_wpa_handshake(self, client_mac):
# Get the corresponding beacon packet of the captured handshake
try:
bssid = self.wpa_handshakes[client_mac]['bssid']
if bssid in self.broadcasted_bssids and not self.wpa_handshakes[
client_mac]['beacon']:
beacon = self.broadcasted_bssids[bssid][1]
self.wpa_handshakes[client_mac]['beacon'] = True
self.wpa_handshakes[client_mac][
'ssid'] = self.broadcasted_bssids[bssid][0]
self.wpa_handshakes[client_mac]['packets'].append(beacon)
except Exception as e:
print "Exception Logging WPA Handshake: ", traceback.print_exc()
return
# Only the last 3 frames are needed
# Aircrack needs at least one beacon to know the ESSID
# Cowpatty is default WPA/WPA2 cracker because
# it is still able to crack the password even if ssid is not in packet information
# as long as it is passed as an argument, the ssid is logged in the name of the file
if (self.wpa_handshakes[client_mac]['frame2']
and self.wpa_handshakes[client_mac]['frame3']
and self.wpa_handshakes[client_mac]['frame4']
and self.wpa_handshakes[client_mac]['beacon']
and not self.wpa_handshakes[client_mac]['logged']):
print "[+] WPA Handshake found for client '{}' and network '{}'\n".format(
client_mac, self.wpa_handshakes[client_mac]['ssid'])
if "wpa_handshakes" not in os.listdir(self.log_dir):
os.makedir(self.log_dir + "wpa_handshakes")
log_file_path = self.log_dir + "wpa_handshakes/handshake_{}_{}.cap".format(
self.wpa_handshakes[client_mac]['ssid'], client_mac)
self._log_packets(log_file_path, client_mac)
def predict(self, model, img_path, true_label, save_img=True):
img = cv2.imread(img_path)
result = img.copy()
img = cv2.resize(img, (32, 32))
img = exposure.equalize_adapthist(img, clip_limit=0.1)
img = img.astype('float32') / 255.
img = np.expand_dims(img, axis=0)
preds = model.predict(img)
j = preds.argmax(axis=1)[0]
pred_label = self.label_names[j]
if j == true_label:
ans = 'correct'
else:
ans = 'incorrect'
if save_img == True:
result = cv2.resize(result, (256, 256))
cv2.rectangle(result, (0, 0), (256, 20), (255, 255, 255), -1)
cv2.putText(result, pred_label + ' (' + ans + ')', (5, 15),
cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 0, 0), 2)
if not os.path.exists('./prediction/'):
os.makedir('./prediction/')
result_path = './prediction/predicted_' + img_path.split('/')[-1]
cv2.imwrite(result_path, result)
return ans, pred_label
def write_to_csv(pid, process_name, csv_data):
#pass
#print csv_data
#file name format : procname_pid_time.csv
process_name = "zerovm"
time_format = '%Y%m%d_%H%M%S'
current_time = time.strftime(time_format)
file_extension = "csv"
directory_name = "csv_report"
if not os.path.exists(directory_name):
os.makedir(directory_name)
output_file_name = "{}_{}_{}.{}".format( process_name,\
pid, current_time,\
file_extension)
'''
output_file_name = process_name +\
"_" + pid + "_" + \
current_time.strftime(time_format)\
+ ".csv"
'''
#print "csv output to: " + output_file_name
logging.debug("csv output to: {}".format(output_file_name))
with open(directory_name + "/" + output_file_name, 'w') as fp:
a = csv.writer(fp, delimiter=',')
a.writerows(csv_data)
def makedir_local(self,path): if not os.path.exists(path): # no need to make a dir that exists os.makedirs(path) elif os.path.isfile(path): # if there is a file there ditch it os.remove(path) del self.files[path] os.makedir(path)
def check_dir(self):
if os.path.exists("data"):
data_con = os.listdir("data")
if "usr" not in data_con:
os.makedir("data/usr")
else:
os.makedirs("data/usr")
print("data folders created...")
def save_page(directory, url, contents):
if not os.path.exists(directory):
os.makedir(directory)
try:
f = file(directory + "/" + urllib2.quote(url, ' '), 'w')
f.write(contents)
f.close()
except Exception, e:
log.error('Error in saving url ' + url + ', Reason: ' + str(e))
def __init__(self, file_name, read, sleep_period=0.01, max_buffer=1000):
scratch_dir = '/scratch/%s/' % os.environ['USER']
if not os.path.exists(scratch_dir):
os.makedir(scratch_dir)
self.scratch_file_name = scratch_dir + str(uuid.uuid4())
self.file_name = file_name
print shutil.copy(self.file_name, self.scratch_file_name)
self.scratch_file = open(self.scratch_file_name, 'r')
self.reader = file_reader(self.scratch_file, read, sleep_period, max_buffer)
def _save_internal():
import json
ndir = os.path.dirname(self.fname)
if not os.path.isdir(ndir):
try:
os.makedir(ndir)
except OSError, e:
if e.errno != 17: # errno 17 is "already exists"
raise
def pre_write(self):
settings = utils.get_settings('ssh')
apt_update(fatal=True)
apt_install(settings['server']['package'])
if not os.path.exists('/etc/ssh'):
os.makedir('/etc/ssh')
# NOTE: don't recurse
utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755,
maxdepth=0)
def __init__(self, file_name, write, sleep_period=0.01):
scratch_dir = '/scratch/%s/' % os.environ['USER']
if not os.path.exists(scratch_dir):
os.makedir(scratch_dir)
self.scratch_file_name = scratch_dir + str(uuid.uuid4())
#self.scratch_file = open(self.scratch_file_name, 'w')
self.writer = file_writer(self.scratch_file_name, write, sleep_period)
self.thread = self.writer.thread
self.file_name = file_name
def check_folders():
""" Check to see if all the directories are present
Add folders here if needed for a new directory """
folders = ("data/KantoBot/", "data/KantoBot/pc/")
for folder in folders:
if not os.path.exsist(folder):
print("Building the directory " + folder + " ...")
os.makedir(folder)
def convert2wav(directory, sub): # convert non-.wav audio files to .wav files using ffmpeg
# When converting, my standard options are: sampling rate 22050 Hz, 1 channel
audioformats = ['mp3','flac','ogg','aiff','aif']
if not os.path.isdir(directory+sub):
os.makedir(directory+sub)
fileslist = filter(os.path.isfile, os.listdir(directory))
bashCommandlist = filter(None,map(lambda name: 'ffmpeg -i '+ directory + name + ' -ar 22050 -ac 1 '+ directory+sub+ name.split('.')[0]+'.wav' if str(name.split('.')[1]) in audioformats else None, fileslist))
for command in bashCommandlist:
subprocess.check_output(['bash','-c', command])
print '.wav conversion done'
def make_ref(filePaths,cp2kRootDir,generic=0):
if generic:
destDir=join(cp2kRootDir,"tests","QS","test_outputs","generic")
else:
archName=commands.getoutput(join(self.cp2kRoot,"tools",
"get_arch_code"))
destDir=join(cp2kRootDir,"tests","QS","test_outputs",archName)
if not os.access(destDir,os.W_OK): os.makedir(destDir)
for filePath in filePaths:
shutil.copy(filePath,destDir)
def mount(src, dst, options=None):
if os.path.exists(src):
if not os.path.exists(dst):
os.makedir(dst)
if options is None:
args = ("/bin/mount", src, dst)
else:
args = ("/bin/mount", options, src, dst)
rc = subprocess.call(args)
return rc
return
def get_cl_main():
if not os.path.exists('misc/cl_main.html'):
r = requests.get('https://www.craigslist.org/about/sites')
page=r.text
if not os.path.exists('misc'):
os.makedir('misc')
with open('misc/cl_main.html','w') as f:
f.write(page)
else:
with open('misc/cl_main.html','r') as f:
page = f.read()
return page
def traverse_object_tree(obj_tree):
count = 0
current_parent_id = 0
prev_parent_id = 0
for elem in obj_tree:
current_parent_id = elem[2]
if(elem[1]!="" ):
if(leaf(elem,obj_tree,count+1) == False):
if(current_parent_id == prev_parent_id):
if (os.path.exists(gethost(elem[1]))==False):
os.mkdir(gethost(elem[1]))
else:
prev_parent_id = current_parent_id
os.chdir("..")
if(os.path.exists(gethost(elem[3]))==False):
os.makedir(gethost(elem[3]))
os.chdir(gethost(elem[3]))
os.makedir(gethost(elem[1]))
os.chdir(gethost(elem[1]))
host = gethost(elem[1])
connect_state = "close"
if(elem[3]==""):
request = generaterequestwithoutref(elem[1],host,connect_state)
data = sendcap(makestr(request),host,80)
else:
request = generaterequest(elem[1],host,referer,connect_state)
data = sendcap(makestr(request),host,80)
f = open(gethost(elem[1]) + ".htm",'w')
f.write(getcontent(data))
if(leaf(elem,obj_tree,count+1) == True):
host = gethost(elem[1])
referer = elem[3]
connect_state = "close"
if(referer==""):
request = generaterequestwithoutref(elem[1],host,connect_state)
data = sendcap(makestr(request),host,80)
else:
request = generaterequest(elem[1],host,referer,connect_state)
data = sendcap(makestr(request),host,80)
if(current_parent_id!=prev_parent_id):
prev_parent_id = current_parent_id
os.chdir("..")
if(os.path.exists(gethost(elem[3]))==False):
os.mkdir(gethost(elem[3]))
os.chdir(gethost(elem[3]))
print getfilename(elem[1])
f = open(getfilename(elem[1]),'wb')
f.write(getcontent(data))
print ("Count done: " + str(count))
count +=1
def loadLaszlo(self, request=False, refreshOnly=False):
"""- special transform type gets special handling through this function, called via URL after startup
because everything needs to be ready before this sort of subprocessing happens.
- If refreshonly we aren't recompiling the XC in question (so we don't need a request), we're just
computing the embed tag which got wiped out on reload.."""
lib = self.library.raw
output = ""
#either process a single transform or loop through and compile them all.
toscan = []
processed = []
#? REMEMBER, this gets run to compile, but the code after this block still needs to run at server start.
if request.req.args.has_key('make'):
mod, trans = request.req.args['make'].split('.')
try: toscan.append(lib[mod]['object.%s'%(trans)])
except: raise 'the transform %s.object.%s not found.'%(mod, trans)
elif request.req.args.has_key('makeall'):
toscan = self.library.getAllTransformsByKind('laszlo')
else:
raise "Provide either ?make=mod.trans or ?makeall=true to proceed with compilation."
for transform in toscan:
processed.append(transform.name)
mod = transform.modulename
name = transform.atomicname
pathprefix = "_lzx"
basepath = "%s%s/%s"%(setup.basePath, mod, pathprefix)
filepath = "%s%s/%s/%s.lzx"%(setup.basePath, mod, pathprefix, name)
swfpath = "%s%s/%s/%s.swf"%(setup.basePath, mod, pathprefix, name)
swffinalpath = "%s%s/%s.swf"%(setup.basePath, mod, name)
#make the dir if it doesn't exist
try: os.makedir(basepath)
except: pass
#build the file and place it at
starttag = """<object %s.%s="true"/>"""%(mod,name)
output = request.processFragment(starttag, [mod], topmost=True, switchLaszloForFlash=False)
try: chunk = parseString(output)
except: raise "Error in top level output from laszlo app, %s, %s"%(lib[trans].name, str(output))
chunk.documentElement.tagName = 'canvas'
chunk.documentElement.setAttribute('proxied', 'false')
#write out to file, changing the root tag to canvas instead of object (nice little feature)
fp = open(filepath, 'w')
fp.write(chunk.documentElement.toxml())
fp.close()
#build the command and run it, producing an .swf
compileLaszloCommand = """%s;%s %s"""%(setup.javaEnvironment, setup.lzcPath, filepath)
try:
run = Popen3(compileLaszloCommand, capturestderr=True)
run.wait()
output = run.fromchild.read()
if output: raise Exception, output
else: os.rename(swfpath,swffinalpath)
def makeDirs(dir, enforce=True):
'''创建文件夹，若存在则忽略
Args:
dir: 需创建的文件夹路径
enforce: 是否强制创建中间路径，默认开启
'''
if checkDir(dir):
pass
elif enforce:
os.makedirs(dir)
else:
os.makedir(dir)
def file_writer(file_name, q):
if file_name.endswith('.pkl'):
write = write_to_pkl_file
elif file_name.endswith('.json'):
write = write_to_json_file
else:
print 'No match to file extension'
raise ValueError('No match for file extension!')
scratch_dir = '/scratch/%s/' % os.environ['USER']
if os.path.exists('/scratch/'):
if not os.path.exists(scratch_dir):
try:
os.makedir(scratch_dir)
except:
pass
is_scratch = os.path.exists(scratch_dir)
is_scratch = False
if is_scratch:
print 'Using scratch'
else:
print 'Not using scratch'
if is_scratch:
scratch_file_name = scratch_dir + str(uuid.uuid4())
print file_name
print scratch_file_name
scratch_file = open(scratch_file_name, 'w')
print 'Opening scratch file'
write_file = scratch_file
else:
write_file = open(file_name, 'w')
print 'Opening file'
while True:
try:
elems = q.get(True, 0.1)
if elems is Queue.Empty or elems is None or elems == '':
print 'Ending writer thread'
write_file.close()
if is_scratch:
print shutil.move(scratch_file_name, file_name)
return
write(write_file, elems)
except Queue.Empty as e:
#print 'Empty'
pass
except Exception as e:
print 'Got other exception'
print e
raise
def init():
if not isdir(out_dir):
os.makedir(out_dir)
if isfile(out):
os.remove(out)
if isfile(avg):
os.remove(avg)
with open("date.cfg") as f:
for date in f:
date_list.append(date.strip())
with open("act.cfg") as f:
for act in f:
act_list.append(act.strip())
def create_dir(path, package_name, opts):
"""Create a directory for a package.
"""
if not os.path.isdir(path):
if opts.dryrun:
print('Would create folder %s.' % path)
return
else:
print('Creating folder for %s package' % package_name)
os.makedir(path)
def download(self):
''' Actually download the files '''
# Create the target local directory if not existing
if not os.path.isdir(RAW_DATA_DIR):
os.makedir(RAW_DATA_DIR)
# Loop thru pages
for xl_file in self.get_files(200):
# Get URL, name, and local path for the files
xl_file_url = self.base_url + xl_file
xl_file_name = xl_file.split('/')[-1]
xl_file_local = os.path.join(RAW_DATA_DIR, xl_file_name)
print >>sys.stdout, "Downloading %s..." % xl_file_url
# Actually retrieve the files
urllib.urlretrieve(xl_file_url, xl_file_local)
def ftp_down(self,remotedir,localdir,filename):
try:
if not os.path.exists(localdir):
os.makedir(localdir)
localfile = localdir + '/' + filename
file = open(localfile,'wb').write
print 'cd' + remotedir
self.__ftp.cwd(remotedir)
print 'download filename=%s/%s' %(remotedir,filename)
self.__ftp.retrbinary("RETR %s" % filename,file,CONST_BUFFER_SIZE)
return True
except ftplib.error_perm:
raise e
return False
def download_each_item(thread_name,socket,elem,obj_tree,connect_s):
current_parent_id = elem[2]
if(elem[1]!="" ):
if(leaf(elem,obj_tree,count+1) == False):
if(current_parent_id == prev_parent_id):
if (os.path.exists(gethost(elem[1]))==False):
os.mkdir(gethost(elem[1]))
else:
prev_parent_id = current_parent_id
os.chdir("..")
if(os.path.exists(gethost(elem[3]))==False):
os.makedir(gethost(elem[3]))
os.chdir(gethost(elem[3]))
os.makedir(gethost(elem[1]))
os.chdir(gethost(elem[1]))
host = gethost(elem[1])
connect_state = "close"
if(elem[3]==""):
request = generaterequestwithoutref(elem[1],host,connect_state)
data = sendcapthread(thread_name,socket,makestr(request),host,80,connect_s)
else:
request = generaterequest(elem[1],host,referer,connect_state)
data = sendcapthread(thread_name,socket,makestr(request),host,80,connect_s)
f = open(gethost(elem[1]) + ".htm",'w')
f.write(getcontent(data))
if(leaf(elem,obj_tree,count+1) == True):
host = gethost(elem[1])
referer = elem[3]
connect_state = "close"
if(referer==""):
request = generaterequestwithoutref(elem[1],host,connect_state)
data = sendcapthread(thread_name,socket,makestr(request),host,80,connect_s)
else:
request = generaterequest(elem[1],host,referer,connect_state)
data = sendcapthread(thread_name,socket,makestr(request),host,80,connect_s)
if(current_parent_id!=prev_parent_id):
prev_parent_id = current_parent_id
os.chdir("..")
if(os.path.exists(gethost(elem[3]))==False):
os.mkdir(gethost(elem[3]))
os.chdir(gethost(elem[3]))
print getfilename(elem[1])
f = open(getfilename(elem[1]),'wb')
f.write(getcontent(data))
print ("Count done: " + str(count))
count +=1
def settings_generator(universe = 82, username = '******', password = '******', flag=False):
path = os.path.normcase('settings/settings.ini')
if (os.path.isfile('settings/settings.ini')):
print("settings file found, stopping now.")
return
if (universe == 82 and not(flag)) or (username == 'defaultName') or (password == 'defaultPassword'):
print("Not all fields specified, fallback on default configuration")
universe = 82
username = '******'
password = '******'
if not (os.path.isdir('settings')):
os.makedir('settings')
with open(path,'w') as foo:
foo.write('[credentials]\nuniverse = '+ str(universe) +'\npassword = '******'\nusername = '+username)
print("Settings.ini generated")
