def fit(self, dataset, dirty_train): index_train_path = utils.get_dir(dataset, 'raw', 'idx_train.csv') index_test_path = utils.get_dir(dataset, 'raw', 'idx_test.csv') dirty_train_path = utils.get_dir(dataset, 'raw', 'dirty_train.csv') dirty_test_path = utils.get_dir(dataset, 'raw', 'dirty_test.csv') index_train = pd.read_csv(index_train_path).values.reshape(-1) index_test = pd.read_csv(index_test_path).values.reshape(-1) ind_path = utils.get_dir(dataset, 'raw', 'AutoER.csv') autoer_result = pd.read_csv(ind_path).values.reshape(-1) ind_train = autoer_result[index_train] ind_test = autoer_result[index_test] dirty_train = pd.read_csv(dirty_train_path) dirty_test = pd.read_csv(dirty_test_path) train_mv = dirty_train.isnull().values.any(axis=1) test_mv = dirty_test.isnull().values.any(axis=1) ind_train = ind_train[train_mv == False] ind_test = ind_test[test_mv == False] ind_train = pd.DataFrame(ind_train.reshape(-1, 1), columns=["label"]) ind_test = pd.DataFrame(ind_test.reshape(-1, 1), columns=["label"]) self.ind_train = ind_train.duplicated(keep="first").values self.ind_test = ind_test.duplicated(keep="first").values
def main(): if request.method == 'POST': # Get the name of the uploaded images uploaded_files = request.files.getlist('image[]') scene_id = str(uuid.uuid4()) scene_dir = get_dir(DATA_DIR, scene_id) upload_dir = get_dir(scene_dir, 'upload') result_dir = get_dir(scene_dir, 'result') for fi, file in enumerate(uploaded_files): # Save the file if file and allowed_file(file.filename): _, file_extension = osp.splitext(file.filename) view_id = str(fi) file.save(osp.join(upload_dir, view_id + cfg.EXT)) # Schedule a task to backend task = osp.join(TASK_DIR, scene_id) open(task, 'wb').write('') while osp.exists(task): time.sleep(cfg.DELAY) filenames = os.listdir(result_dir) return render_template('result.html', scene_id=scene_id, filenames=filenames) return render_template('main.html')
def init(dataset, test_ratio=0.3, seed=1, max_size=None): """ Initialize dataset: raw -> dirty -> dirty_train, dirty_test Args: dataset (dict): dataset dict in config.py max_size (int): maximum limit of dataset size test_ratio: ratio of test set seed: seed used to split dataset """ print("Initialize dataset {}".format(dataset['data_dir'])) # load raw data raw_path = utils.get_dir(dataset, 'raw', 'raw.csv') raw = pd.read_csv(raw_path) # delete missing labels or all missing values if 'missing_values' not in dataset['error_types']: dirty = delete_missing_values(raw) else: dirty = delete_missing_labels(raw, dataset['label']) # split dataset train, test, idx_train, idx_test = split(dirty, test_ratio, seed, max_size) # save train / test save_path_pfx = utils.get_dir(dataset, 'raw', 'dirty') utils.save_dfs(train, test, save_path_pfx) # save the version (seed) of dataset utils.save_version(save_path_pfx, seed) # save index save_path_pfx = utils.get_dir(dataset, 'raw', 'idx') utils.save_dfs(idx_train, idx_test, save_path_pfx)
def preprocess(dataset, error_type, train_file, normalize=True, down_sample_seed=1): """Load and preprocess data Args: dataset (dict): dataset dict in config error_type (string): error type train_file (string): prefix of file of training set normalize (bool): whehter to standarize the data down_sample_seed: seed for down sampling """ # check train and test version are consistent check_version(dataset, error_type, train_file) # get path of train file and test files train_path = utils.get_dir(dataset, error_type, train_file + "_train.csv") test_files = utils.get_test_files(error_type, train_file) test_path_list = [ utils.get_dir(dataset, error_type, test_file + "_test.csv") for test_file in test_files ] # load data X_train, y_train, X_test_list, y_test_list = load_data( dataset, train_path, test_path_list) ## preprocess data # drop irrelavant features if "drop_variables" in dataset.keys(): drop_columns = dataset['drop_variables'] drop_variables(X_train, X_test_list, drop_columns) # down sample if imbalanced if "class_imbalance" in dataset.keys() and dataset["class_imbalance"]: X_train, y_train = down_sample(X_train, y_train, down_sample_seed) # encode label if dataset['ml_task'] == 'classification': y_train, y_test_list = encode_cat_label(y_train, y_test_list) # text embedding if "text_variables" in dataset.keys(): text_columns = dataset["text_variables"] X_train, X_test_list = encode_text_features(X_train, X_test_list, y_train, text_columns) # encode categorical features X_train, X_test_list = encode_cat_features(X_train, X_test_list) # normalize data if normalize: scaler = StandardScaler() X_train = scaler.fit_transform(X_train) X_test_list = [scaler.transform(X_test) for X_test in X_test_list] return X_train, y_train, X_test_list, y_test_list, test_files
def fit(self, dataset, dirty_train): index_train_path = utils.get_dir(dataset, 'raw', 'idx_train.csv') index_test_path = utils.get_dir(dataset, 'raw', 'idx_test.csv') index_train = pd.read_csv(index_train_path).values.reshape(-1) index_test = pd.read_csv(index_test_path).values.reshape(-1) clean_path = utils.get_dir(dataset, 'raw', 'mislabel_clean_raw.csv') clean = utils.load_df(dataset, clean_path) self.clean_train = clean.loc[index_train, :] self.clean_test = clean.loc[index_test, :]
def check_version(dataset, error_type, train_file): """Check whether train and test are of the same version""" train_path_pfx = utils.get_dir(dataset, error_type, train_file) train_version = utils.get_version(train_path_pfx) test_files = utils.get_test_files(error_type, train_file) for test_file in test_files: test_path_pfx = utils.get_dir(dataset, error_type, test_file) test_version = utils.get_version(test_path_pfx) assert (train_version == test_version)
def __init__(self, save_dir): self.save_dir = utils.get_dir(save_dir) self.model_save_dir = utils.get_dir( os.path.join(self.save_dir, 'models')) self.summary_save_dir = utils.get_dir( os.path.join(self.save_dir, 'summaries')) self.image_save_dir = utils.get_dir( os.path.join(self.save_dir, 'images')) self.log_save_dir = utils.get_dir(os.path.join(self.save_dir, 'logs'))
def fit(self, dataset, df): clean_raw_path = utils.get_dir(dataset, 'raw', 'Holoclean_mv_clean.csv') clean_raw = pd.read_csv(clean_raw_path) index_train_path = utils.get_dir(dataset, 'raw', 'idx_train.csv') index_test_path = utils.get_dir(dataset, 'raw', 'idx_test.csv') index_train = pd.read_csv(index_train_path).values.reshape(-1) index_test = pd.read_csv(index_test_path).values.reshape(-1) self.clean_train = clean_raw.iloc[index_train, :] self.clean_test = clean_raw.iloc[index_test, :]
def set_save_name(name): """ Edits all constants dependent on SAVE_NAME. @param name: The new save name. """ global SAVE_NAME, MODEL_SAVE_DIR, SUMMARY_SAVE_DIR, IMG_SAVE_DIR SAVE_NAME = name MODEL_SAVE_DIR = utils.get_dir(os.path.join(SAVE_DIR, SAVE_NAME, 'Models')) SUMMARY_SAVE_DIR = utils.get_dir( os.path.join(SAVE_DIR, SAVE_NAME, 'Summaries')) IMG_SAVE_DIR = utils.get_dir(os.path.join(SAVE_DIR, SAVE_NAME, 'Images')) print('Set save dir to {}'.format(os.path.join(SAVE_DIR, SAVE_NAME)))
def get_img(img, size=None): path = os.path.join(utils.get_dir(__file__), "img", img) img = PIL.Image.open(path) if size and (size < img.size): log.debug("Resizing %s; requested size: %s, orig image size: %s", img.filename, size, img.size) img = img.resize(size, PIL.Image.LANCZOS) return PIL.ImageTk.PhotoImage(img)
def out_as_csv(self, filename): print(len(self.gbid_dic)) newdir = utils.get_dir('out') out_name = os.path.join(newdir, filename) days_list = list(self.days.keys()) days_list.sort() csv = csv_output.CSVOutPut() for i, day in enumerate(days_list): day_dict = self.days[day] dur_dict = {} csv.set(0, i * 2, day) csv.set(0, i * 2 + 1, day) for uk, lo in day_dict.items(): dur_dict[lo.get_duration()] = dur_dict.get( lo.get_duration(), 0) + 1 dur_list = list(dur_dict.keys()) dur_list.sort() for j in range(121): csv.set(j + 1, i * 2, j) csv.set(j + 1, i * 2 + 1, dur_dict.get(j, 0)) # for j, dur in enumerate(dur_list): # count = dur_dict[dur] # csv.set(j + 1, i * 2, dur) # csv.set(j + 1, i * 2 + 1, count) csv.output(out_name)
def main(argv=None): save_dir = utils.get_dir(FLAGS.save_dir) file_paths, labels = load_pathnlabel(FLAGS.annofile_path, abs_path=FLAGS.abs_path) assert len(file_paths) == len(labels) wrtie_videoframes_to_tfrecord(file_paths, labels, save_dir, frame_limit=FLAGS.min_len)
def fit(self, dataset, dirty_train): dirty_raw_path = utils.get_dir(dataset, 'raw', 'raw.csv') clean_raw_path = utils.get_dir(dataset, 'raw', 'inconsistency_clean_raw.csv') if not os.path.exists(clean_raw_path): print( "Must provide clean version of raw data for cleaning inconsistency" ) sys.exit(1) dirty_raw = utils.load_df(dataset, dirty_raw_path) clean_raw = utils.load_df(dataset, clean_raw_path) N, m = dirty_raw.shape dirty_raw = dirty_raw.values clean_raw = clean_raw.values mask = (dirty_raw != clean_raw) dirty = dirty_raw[mask] clean = clean_raw[mask] self.incon_dict = dict(zip(dirty, clean))
def clean_error(dataset, error): """ Clean one error in the dataset. Args: dataset (dict): dataset dict in dataset.py error (string): error type """ # create saving folder save_dir = utils.get_dir(dataset, error, create_folder=True) # load dirty data dirty_path_pfx = utils.get_dir(dataset, 'raw', 'dirty') dirty_train, dirty_test, version = utils.load_dfs(dataset, dirty_path_pfx, return_version=True) # delete missing values if error type is not missing values if error != 'missing_values': dirty_train = dirty_train.dropna().reset_index(drop=True) dirty_test = dirty_test.dropna().reset_index(drop=True) # save dirty data dirty_path_pfx = os.path.join(save_dir, 'dirty') utils.save_dfs(dirty_train, dirty_test, dirty_path_pfx, version) # clean the error in the dataset with various cleaning methods error_type = utils.get_error(error) for clean_method, cleaner in error_type['clean_methods'].items(): print( " - Clean the error with method '{}'".format(clean_method)) # fit on dirty train and clean both train and test cleaner.fit(dataset, dirty_train) clean_train, ind_train, clean_test, ind_test = cleaner.clean( dirty_train, dirty_test) # save clean train and test data clean_path_pfx = os.path.join(save_dir, clean_method) utils.save_dfs(clean_train, clean_test, clean_path_pfx, version) # save indicator ind_path_pfx = os.path.join(save_dir, 'indicator_{}'.format(clean_method)) utils.save_dfs(ind_train, ind_test, ind_path_pfx)
def wrtie_videoframes_to_tfrecord(file_paths, labels, save_dir, frame_limit=None): assert len(file_paths) == len(labels), 'Files and labels mismatch' print 'Total # of files {:d}'.format(len(file_paths)) save_dir = utils.get_dir(save_dir) # filter away if frame_limit: filtered_pairs = [(f, l) for f, l in zip(file_paths, labels) if len(glob.glob(os.path.join(f,'*.{:s}'.format(FLAGS.image_format)))) >= frame_limit] file_paths, labels = zip(*filtered_pairs) assert len(file_paths) == len(labels), 'Checking filtering' n_files = len(file_paths) print '# of valid videos: {:d}'.format(n_files) for i, s_label, s_filename in zip(range(n_files), labels, file_paths): filename_stem = s_filename.split(os.sep)[-1] print 'Processing {:d} | {:d} \t {:s}'.format(i, n_files, filename_stem) file_list = glob.glob(os.path.join(s_filename, '*.{:s}'.format(FLAGS.image_format))) n_images = len(file_list) images = [] tf_save_name = os.path.join(save_dir, '{:s}.{:s}'.format(filename_stem, FLAGS.tf_format)) writer = tf.python_io.TFRecordWriter(tf_save_name) for single_filename in file_list: img = imread(single_filename, mode='RGB') images.append(img) np_images = np.array(images).astype(np.uint8) seq_shape = np_images.shape seq_d = seq_shape[0] # depth, length of seq assert n_images==seq_d seq_h = seq_shape[1] # height seq_w = seq_shape[2] # width seq_c = seq_shape[3] # channels image_raw = np_images.tostring() example = tf.train.Example(features=tf.train.Features(feature={ 'd': _int64_feature(seq_d), 'h': _int64_feature(seq_h), 'w': _int64_feature(seq_w), 'c': _int64_feature(seq_c), 'label': _int64_feature(int(s_label)), 'image': _bytes_feature(image_raw), 'filename': _bytes_feature(filename_stem)})) writer.write(example.SerializeToString()) print 'Done'
def filter_sys_log(tag_name): new_dir = utils.get_dir('tmp') basename = os.path.basename(const.SYS_LOG_NAME) fw_name = os.path.join(new_dir, '{}.{}.log'.format(basename, tag_name)) fw = utils.utf8_open(fw_name, 'w') with utils.utf8_open(const.SYS_LOG_NAME, encoding='utf-8') as fr: for line in fr: if tag_name in line: fw.write(line) fw.close() return fw_name
def fit(self, dataset, df): clean_raw_path = utils.get_dir(dataset, 'raw', 'Holoclean_outlier_clean.csv') index_train_path = utils.get_dir(dataset, 'raw', 'idx_train.csv') index_test_path = utils.get_dir(dataset, 'raw', 'idx_test.csv') index_train = pd.read_csv(index_train_path).values.reshape(-1) index_test = pd.read_csv(index_test_path).values.reshape(-1) clean_raw = pd.read_csv(clean_raw_path) if 'missing_values' in dataset['error_types']: dirty_train = pd.read_csv( utils.get_dir(dataset, 'raw', 'dirty_train.csv')) dirty_test = pd.read_csv( utils.get_dir(dataset, 'raw', 'dirty_test.csv')) raw = pd.read_csv(utils.get_dir(dataset, 'raw', 'raw.csv')) raw_mv_rows = raw.isnull().values.any(axis=1) train_mv_rows = dirty_train.isnull().values.any(axis=1) test_mv_rows = dirty_test.isnull().values.any(axis=1) old_index = np.arange(len(raw))[raw_mv_rows == False] new_index = np.arange(len(raw) - sum(raw_mv_rows)) index_map = {} for o, n in zip(old_index, new_index): index_map[o] = n index_train_no_mv = index_train[train_mv_rows == False] index_test_no_mv = index_test[test_mv_rows == False] index_train = [index_map[i] for i in index_train_no_mv] index_test = [index_map[i] for i in index_test_no_mv] self.clean_train = clean_raw.iloc[index_train, :] self.clean_test = clean_raw.iloc[index_test, :]
def __init__(self): self.captcha = Captcha() self.text, image = self.captcha.get_captcha_text_and_image() self.IMAGE_HEIGHT = 60 self.IMAGE_WIDTH = 160 self.MAX_CAPTCHA = len(self.text) self.CHAR_SET_LEN = self.captcha.get_len_char_set() self.X = tf.placeholder(tf.float32, [None, self.IMAGE_HEIGHT * self.IMAGE_WIDTH]) self.Y = tf.placeholder(tf.float32, [None, self.MAX_CAPTCHA * self.CHAR_SET_LEN]) self.keep_prob = tf.placeholder(tf.float32) self.model_path = is_dir(get_dir() + "ckpt" + os.sep) self.output = self.creat_captcha_cnn()
def filter_guild_train(): tmp_dir = utils.get_dir('tmp') fw_name = os.path.join(tmp_dir, 'guild_train_tlog.log') if os.path.exists(fw_name): return fw_name fw = utils.utf8_open(fw_name, 'w') for line in utils.get_origin_line_stream(): if 'guild train upgrade' in line and line.startswith( 'LOG_GUILD_CONTRIBUTION'): fw.write(line) fw.close() return fw_name
def reset(dataset): """ Reset dataset""" # delete folders for each error for error in dataset['error_types']: utils.remove(utils.get_dir(dataset, error)) # delete dirty_train and dirty_test in raw folder utils.remove(utils.get_dir(dataset, 'raw', 'dirty_train.csv')) utils.remove(utils.get_dir(dataset, 'raw', 'dirty_test.csv')) utils.remove(utils.get_dir(dataset, 'raw', 'dirty.csv')) utils.remove(utils.get_dir(dataset, 'raw', 'idx_train.csv')) utils.remove(utils.get_dir(dataset, 'raw', 'idx_test.csv')) utils.remove(utils.get_dir(dataset, 'raw', 'version.json'))
def __init__(self, color, seed, x, y, size=36): self.seed = seed self.generator = random.Random(seed) self.color = color self.size = size self.sprites = sprite.RenderUpdates() self.sprite = sprite.Sprite() self.dirName = get_dir() self._path = self.dirName + "ghost_" + self.color + ".png" img = pygame.image.load(self._path) self.sprite.image = pygame.transform.scale(img, (self.size, self.size)) self.sprites.add(self.sprite) self.position = (x, y) self.startposition = self.position self.reset()
def out_as_csv(self, filename): out_dir = utils.get_dir('out') out_name = os.path.join(out_dir, filename) csv = csv_output.CSVOutPut() index = 0 for day, day_dict in self.uk_day_dict.items(): sum_score = sum(day_dict.values()) csv.set(0, index, day) csv.set(1, index, sum_score / len(day_dict)) csv.set(2, index, sum_score) csv.set(3, index, len(day_dict)) index += 1 csv.output(out_name)
def filter_login_log(): dirname = utils.get_dir('tmp') fw_name = utils.get_out_name('tmp', 'log_in_and_out.log') if os.path.exists(fw_name): return fw_name fw = utils.utf8_open(fw_name, 'w', encoding='utf-8') for line in utils.get_origin_line_stream(): if not (line.startswith('SecLogin') or line.startswith('SecLogout')): continue fw.write(line) fw.close() return fw_name
def out_as_csv(self, csv_name): dirname = utils.get_dir('out') full_csv_name = os.path.join(dirname, csv_name) days_list = list(self.days.keys()) days_list.sort() with utils.utf8_open(full_csv_name, 'w') as fw: days_str = ','.join(map(str, days_list)) fw.write(days_str + '\n') avatar_count = ','.join( map(lambda day: str(len(self.days[day])), days_list)) fw.write(avatar_count + '\n') def sum_times(day_dict): return sum(map(len, day_dict.values())) times = ','.join( map(lambda day: str(sum_times(self.days[day])), days_list)) fw.write(times + '\n')
def download(json_data, save_dir, max_len=100000, min_len=-1, video_format='mp4', video_resolution='360p', dstfile=None): save_dir = utils.get_dir(save_dir) count = 0 nfiles = len(json_data) testids = [] testlabels = [] with open(dstfile, 'w') as f: for i, datum in enumerate(json_data): if datum['duration'] > min_len and datum['duration'] < max_len: videolink = youtube_link + datum['id'] try: yt = YouTube(videolink) video = yt.get(video_format, video_resolution) video.filename = datum['id'] video.download(save_dir) count += 1 print('{:d} : {:d} -- {:s}, len: {:d}'.format( i, nfiles, datum['id'], int(datum['duration']))) videoname = os.path.join( save_dir, '{:s}.{:s}'.format(datum['id'], video_format)) videolabel = datum['label487'] f.write('{:s} {:s}\r\n'.format( videoname, ' '.join(map(str, videolabel)))) f.flush() except: print('{:s} NOT valid'.format(datum['id'])) continue print('Done Download, {:d} is downloaded, information saved to '.format( count))
def download(save_dir, rewrite=False): save_dir = utils.get_dir() if rewrite: utils.clear_dir(save_dir) filename = DATA_URL.split(os.pathsep)[-1] filepath = os.path.join(save_dir, filename) if not os.path.isfile(filepath): def _progress(count, block_size, total_size): sys.stdout.write('\r>> Downloading %s %.1f%%' % (filename, float(count * block_size) / float(total_size) * 100.0)) sys.stdout.flush() filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress) print() statinfo = os.stat(filepath) print('Successfully downloaded', filename, statinfo.st_size, 'bytes.') tarfile.open(filepath, 'r:gz').extractall(save_dir)
def inject(dataset): """ Inject mislabels Args: dataset (dict): dataset dict in config """ # create saving folder major_save_dir = utils.makedirs( [config.data_dir, dataset["data_dir"] + "_major", 'raw']) minor_save_dir = utils.makedirs( [config.data_dir, dataset["data_dir"] + "_minor", 'raw']) uniform_save_dir = utils.makedirs( [config.data_dir, dataset["data_dir"] + "_uniform", 'raw']) # load clean data clean_path = utils.get_dir(dataset, 'raw', 'raw.csv') clean = utils.load_df(dataset, clean_path) clean = clean.dropna().reset_index(drop=True) major_clean_path = os.path.join(major_save_dir, 'mislabel_clean_raw.csv') minor_clean_path = os.path.join(minor_save_dir, 'mislabel_clean_raw.csv') uniform_clean_path = os.path.join(uniform_save_dir, 'mislabel_clean_raw.csv') clean.to_csv(major_clean_path, index=False) clean.to_csv(minor_clean_path, index=False) clean.to_csv(uniform_clean_path, index=False) label = dataset['label'] # uniform flip uniform = uniform_class_noise(clean, label) # pairwise flip major, minor = pairwise_class_noise(clean, label) major_raw_path = os.path.join(major_save_dir, 'raw.csv') minor_raw_path = os.path.join(minor_save_dir, 'raw.csv') uniform_raw_path = os.path.join(uniform_save_dir, 'raw.csv') major.to_csv(major_raw_path, index=False) minor.to_csv(minor_raw_path, index=False) uniform.to_csv(uniform_raw_path, index=False)
def __init__(self, args): self._make_dataset(args) self._make_model(args) tc.manual_seed(args.seed) if args.cuda and tc.cuda.is_available(): tc.cuda.manual_seed_all(args.seed) if tc.cuda.device_count() > 1: self.batch_size = args.batch_size * tc.cuda.device_count() self.model = DataParallel(self.model) else: self.batch_size = args.batch_size self.model = self.model.cuda() else: self.batch_size = args.batch_size self._make_optimizer(args) self._make_loss(args) self._make_metric(args) self.num_training_samples = args.num_training_samples self.tag = args.tag or 'default' self.dump_dir = get_dir(args.dump_dir) self.train_logger = get_logger('train.{}.{}'.format( self.__class__.__name__, self.tag))
def main(argv): FLAGS = gflags.FLAGS gflags.DEFINE_string('srcfile', 'json/sports1m_test.json', 'jason file to read from[json/sports1m_test.json]') gflags.DEFINE_string('savedir', 'tmp', 'dstination directory to save files[tmp]') gflags.DEFINE_string('video_format', 'mp4', 'video format[mp4]') gflags.DEFINE_string('video_resolution', '360p', 'video resolution 360p, 720p and so on[360p]') gflags.DEFINE_integer('min_len', 4, 'Minimal length(4 seconds)') gflags.DEFINE_integer('max_len', 30, 'Maximal length (30 seconds)') gflags.DEFINE_string( 'dstfile', None, 'dstination json file[json/sports1m_test_(duration).json]') gflags.DEFINE_boolean('rewrite', True, 'rewrite everything in saved dir[True]') argv = FLAGS(argv) srcfile = FLAGS.srcfile dstfile = FLAGS.dstfile savedir = os.path.abspath(FLAGS.savedir) savedir = utils.get_dir(savedir) if FLAGS.rewrite: utils.clear_dir(savedir) raw_json_data = read_json(srcfile) if not dstfile: srcname, srcext = os.path.splitext(srcfile) dstfile = '{:s}-{:02d}-{:02d}{:s}'.format(srcname, FLAGS.min_len, FLAGS.max_len, '.txt') download(raw_json_data, save_dir=savedir, dstfile=dstfile, max_len=FLAGS.max_len, min_len=FLAGS.min_len)
def out_as_csv(self, filename): csv = csv_output.CSVOutPut() out_name = os.path.join(utils.get_dir('out'), filename) day_list = list(self.days.keys()) day_list.sort() for index, day in enumerate(day_list): csv.set(0, index * 2, str(day)) csv.set(0, index * 2 + 1, str(day)) day_dict = self.days[day] levelDict = {} for unique_key, lo in day_dict.items(): levelDict.setdefault(lo.level, []) levelDict[lo.level].append(lo) levelList = list(levelDict.keys()) levelList.sort() for _index, level in enumerate(levelList): num = len(levelDict[level]) csv.set(_index + 1, index * 2, str(level)) csv.set(_index + 1, index * 2 + 1, str(num)) csv.output(out_name)
def __init__(self, buddy, startx, starty, color = 'green', size=36): self.buddy = buddy self.nick = buddy.props.nick self.score = 0 self.color = color self.size = size self.startposition = (startx, starty) self.hidden = False self.bonusplayers = None self.opensprites = sprite.RenderUpdates() self.closedsprites = sprite.RenderUpdates() self.dirName = get_dir() self.open = sprite.Sprite() self.closed = sprite.Sprite() self._path = self.dirName + "pacman-" + self.color img = pygame.image.load(self._path + "-open-right.png") self.open.image = pygame.transform.scale(img, (self.size, self.size)) img = pygame.image.load(self._path + "-closed.png") self.closed.image = pygame.transform.scale(img, (self.size, self.size)) self.opensprites.add(self.open) self.closedsprites.add(self.closed) self.mouthClosed = 1 self.supertime = 0 self.reset()
def get_config_dir(self): return utils.get_dir(self.config_file)
def _do_read_config(self, config_file, pommanipext): """Reads config for a single job defined by section.""" parser = InterpolationConfigParser() dataset = parser.read(config_file) if config_file not in dataset: raise IOError("Config file %s not found." % config_file) if parser.has_option('common','include'): include = parser.get('common', 'include') if include is not "": sections_ = self.read_and_load(include) for section_ in sections_: if parser.has_section(section_): raise DuplicateSectionError( "The config section [%s] is existed in %s and include %s cfg file" % ( section_, config_file, re.split("\\s+", include.strip())[1])) parser._sections.update(sections_) pom_manipulator_config = {} common_section = {} package_configs = {} if pommanipext and pommanipext != '' and pommanipext != 'None': #TODO ref: remove none check, it is passed over cmd line in jenkins build parse_pom_manipulator_ext(pom_manipulator_config, parser, pommanipext) if not parser.has_section('common'): logging.error('Mandatory common section missing from configuration file.') raise NoSectionError, 'Mandatory common section missing from configuration file.' common_section['tag'] = parser.get('common', 'tag') common_section['target'] = parser.get('common', 'target') common_section['jobprefix'] = parser.get('common', 'jobprefix') common_section['jobciprefix'] = parser.get('common', 'jobciprefix') common_section['jobjdk'] = parser.get('common', 'jobjdk') if parser.has_option('common', 'mvnver'): common_section['mvnver'] = parser.get('common', 'mvnver') if parser.has_option('common', 'skiptests'): common_section['skiptests'] = parser.get('common', 'skiptests') if parser.has_option('common', 'base'): common_section['base'] = parser.get('common', 'base') if parser.has_option('common', 'citemplate'): common_section['citemplate'] = parser.get('common', 'citemplate') if parser.has_option('common', 'jenkinstemplate'): common_section['jenkinstemplate'] = parser.get('common', 'jenkinstemplate') if parser.has_option('common', 'product_name'): common_section['product_name'] = parser.get('common', 'product_name') if parser.has_option('common', 'include'): common_section['include'] = parser.get('common', 'include') common_section['jobfailureemail'] = parser.get('common', 'jobfailureemail') config_dir = utils.get_dir(config_file) #Jira if parser.has_option('common', 'shared_config') and parser.get('common', 'shared_config') is not "": parse_shared_config(common_section, config_dir, parser) common_section['jobtimeout'] = parser.getint('common', 'jobtimeout') common_section['options'] = {} # If the configuration file has global properties insert these into the common properties map. # These may be overridden later by particular properties. if parser.has_option('common', 'globalproperties'): common_section['options']['properties'] = dict(x.strip().split('=') for x in parser.get('common', 'globalproperties').replace(",\n", ",").split(',')) else: # Always ensure properties has a valid dictionary so code below doesn't need multiple checks. common_section['options']['properties'] = {} # The same for global profiles if parser.has_option('common', 'globalprofiles'): common_section['options']['profiles'] = [x.strip() for x in parser.get('common', 'globalprofiles').split(',')] else: # Always ensure profiles has a valid list so code below doesn't need multiple checks. common_section['options']['profiles'] = [] if os.path.dirname(config_file): config_path = os.path.dirname(config_file) else: config_path = os.getcwd() logging.info("Configuration file is %s and path %s", os.path.basename(config_file), config_path) for section in parser.sections(): config_type = self.read_config_type(parser, section) if section == 'common' or config_type == "bom-builder-meta": logging.debug ('Skipping section due to meta-type %s', section) continue self._do_read_section(config_path, os.path.basename(config_file), package_configs, parser, section) return (common_section, package_configs, pom_manipulator_config)
import thread from gi.repository import GObject, Gtk import subprocess import os import time import utils import pbpulse steps = ["Préparation","Installation de LibreOffice","Installation de Firefox"] CMD=os.path.join(utils.get_dir(__file__),"prog.sh") class ThreadScript(thread.Thread): def __init__(self,progress,info,text,action=None): thread.Thread.__init__(self) self._running = False self.progress = progress self.info = info self.text = text self.action = action def run(self): self._running = True self.cmd = subprocess.Popen(CMD,stdout=subprocess.PIPE,universal_newlines=True) while self._running: for line in self.cmd.stdout: line = line[:-1] try: self.progress(float(line)) except ValueError: if line[0] == "#":
def get_log_dir(): logdir = os.path.join(get_dir(__file__), 'log') makedirs(logdir) return logdir
def set_icon(window, icon): this_dir = utils.get_dir(__file__) window.wm_iconbitmap(os.path.join(this_dir, "img", icon))