def create_saving_dirs(self): ### Update name ### self.name += '_MReg%.2fEReg%.2fB%d' % (self.args.magR, self.args.enR, self.args.BS) create_directory('%s/%s' % (self.args.metrics_dir, self.name)) create_directory('%s/%s' % (self.args.model_dir, self.name))
def export_software_profiles(): db_session = DBSession() software_profile_list = request.args.getlist( 'software_profile_list[]')[0].split(",") software_profiles = get_software_profile_list(db_session) d = {"CSM Server:Software Profile": {}} for software_profile in software_profiles: if software_profile.name in software_profile_list: d["CSM Server:Software Profile"][ software_profile.name] = software_profile.packages temp_user_dir = create_temp_user_directory(current_user.username) software_profile_export_temp_path = os.path.normpath( os.path.join(temp_user_dir, "software_profile_export")) create_directory(software_profile_export_temp_path) make_file_writable(software_profile_export_temp_path) with open( os.path.join(software_profile_export_temp_path, 'software_profiles.json'), 'w') as command_export_file: command_export_file.write(json.dumps(d, indent=2)) return send_file(os.path.join(software_profile_export_temp_path, 'software_profiles.json'), as_attachment=True)
def download_system_logs(): db_session = DBSession() logs = db_session.query(Log) \ .order_by(Log.created_time.desc()) contents = '' for log in logs: contents += get_datetime_string(log.created_time) + ' UTC\n' contents += log.level + ':' + log.msg + '\n' if log.trace is not None: contents += log.trace + '\n' contents += '-' * 70 + '\n' # Create a file which contains the size of the image file. temp_user_dir = create_temp_user_directory(current_user.username) log_file_path = os.path.normpath(os.path.join(temp_user_dir, "system_logs")) create_directory(log_file_path) make_file_writable(log_file_path) log_file = open(os.path.join(log_file_path, 'system_logs'), 'w') log_file.write(contents) log_file.close() return send_file(os.path.join(log_file_path, 'system_logs'), as_attachment=True)
def create_mapping_index_page(mapping_file_pathname, template): """Create an index html page for a mapping. e.g. http://airm.aero/developers/fixm-4.2.0-to-airm-1.0.0.html Keyword arguments: mapping_file_pathname -- string defining the location and name of the mapping e.g. data/xlsx/mapping FIXM 4.2.0.xlsx template -- string defining the location and name of the html template e.g. data/html/templates/concept-list-template.html """ import mapping mapping = mapping.Mapping(mapping_file_pathname) mapping_metadata = mapping.metadata mapping_dict = mapping.dictionary import utils utils.create_directory("docs/airm/developers/" + mapping_metadata["url_name"]) soup = utils.create_html_soup(template) soup.title.string = mapping_metadata[ "name"] + " | Semantic Correspondences | AIRM.aero" soup.find(text="MAPPING_NAME_BC").replace_with(mapping_metadata["name"]) soup.find(text="MAPPING_NAME_H2").replace_with(mapping_metadata["name"]) for record in mapping_dict: new_table_row = create_index_table_row(record, mapping_metadata) soup.find('tbody').insert(1, new_table_row) f = open("docs/airm/developers/" + mapping_metadata["url_name"] + ".html", "w+") f.write(soup.prettify()) f.close()
def build_website(path, dist_path, theme_path, theme, collections, statics): website = create_website_model(path, collections) log_default('Loading "%s" theme...', theme) templates = load_templates(theme_path) log_default('Rendering templates...') render_templates(dist_path, templates, website) # add static files if ARGS.static: log_default('Adding static files...') for static_dir in ARGS.static: log_default('\tAdding "%s" folder', static_dir) static_path = os.path.join(path, static_dir) static_dist_path = os.path.join(dist_path, static_dir) for static_file in get_all_files(static_path): log_default('\t\tAdding "%s"...', static_file) dest_path = static_file.replace(static_path, static_dist_path) create_directory(os.path.dirname(dest_path)) shutil.copy2(static_file, dest_path) # copy all non-html files from theme folder for theme_file in get_all_files(theme_path, '.html', negative=True): dest_path = theme_file.replace(theme_path, dist_path) create_directory(os.path.dirname(dest_path)) shutil.copy2(theme_file, dest_path) log_default('Done :thumbs_up:', not_verbose=True)
def get_labels(edges_file, output_dir="", read_cols=None, cols_to_rename=None): ''' Loads the data from a provided file and transforms the data to a dataframe with two columns: "node" and "label" ("node" columns contains sorted distinct node ids). Finally saves data to 'nodes_labels.csv' csv file. edges_file: filepath to a csv file where nodes and hyperrelations are columns and traversed graph paths are rows output_dir: directory where the output dataframe should be saved read_cols: list of columns names to be transofmed from provided files cols_to_rename: dictionary used to rename columns (which are used as labels); useful if several columns contains the same type of nodes e.g. 'g1' and 'g2' both have label 'gene' but appear separately because of specific graph traversion path ''' utils.create_directory(output_dir) usecols = None if read_cols != None: usecols = read_cols raw_df = pd.read_csv(edges_file, usecols=usecols) raw_df = rename_cols(raw_df, cols_to_rename) labels = raw_df.columns enc_dict = encode_columns(labels, output_dir) raw_df = raw_df.rename(columns=enc_dict) raw_df = pd.DataFrame(raw_df.stack()).droplevel(0) raw_df.reset_index(inplace=True) raw_df.columns = ["label", "node"] raw_df = raw_df[["node", "label"]] raw_df["node"] = raw_df["node"].str.replace("V", "") raw_df["node"] = pd.to_numeric(raw_df["node"]) raw_df = raw_df.sort_values(by=["node"]) raw_df = raw_df.drop_duplicates() raw_df.to_csv("{}nodes_labels.csv".format(output_dir), index=False)
async def download_file(_url, folder, file_id=None) -> None: create_directory(folder) if not Path(f"{folder}/{file_id}_{_url}").exists() or getsize(Path(f"{folder}/{file_id}_{_url}"))['raw'] == 0: async with semaphore, aiohttp.ClientSession(json_serialize=ujson.dumps, headers={'Connection': 'keep-alive'}) as session: async with session.get(f"https://telegra.ph/file/{_url}") as response: assert response.status == 200 path = Path(f"{folder}/{file_id}_{_url}") if parser.parse_args().compress: loop = asyncio.get_running_loop() executor = ProcessPoolExecutor() destination = append_extension(str(path), "webp") if not Path(destination).exists() or getsize(destination)['formatted'] == 0: if is_image_by_url(_url): await asyncio.gather(*[loop.run_in_executor( executor, compress_image, await response.read(), Path(destination)) if is_image_by_url(_url) else None]) else: await write_file(await response.read(), path) log(f"[download] {file_id}_{_url} — {getsize(path)['formatted']}") else: await write_file(await response.read(), path) log(f"[download] {file_id}_{_url} — {getsize(path)['formatted']}")
def __init__(self, options): self.ports = options.ports self.password = options.password self.motdfile = options.motd self.verbose = options.verbose self.debug = options.debug self.ldap_server = None self.ldap_port = None self.ldap_dn = '' self.ldap_encryption = None self.ldap_require_cert = None self.ldap_debug = False if ldap_available: self.ldap_server = options.ldap_server self.ldap_port = options.ldap_port self.ldap_dn = options.ldap_dn #self.ldap_password = options.ldap_password self.ldap_encryption = options.ldap_encryption self.ldap_require_cert = options.ldap_require_cert self.ldap_debug = options.ldap_debug self.logdir = options.logdir self.ssl_pem_file = options.ssl_pem_file self.statedir = options.statedir self.name = socket.getfqdn()[:63] # Server name limit from the RFC. self.channels = {} # irc_lower(Channel name) --> Channel instance. self.clients = {} # Socket --> Client instance. self.nicknames = {} # irc_lower(Nickname) --> Client instance. if self.logdir: create_directory(self.logdir) if self.statedir: create_directory(self.statedir)
def download(model_list_path, output_dir, force): model_list = load_document(model_list_path) if model_list is None: sys.exit(1) validate_schema(model_list) target_root = os.path.join(output_dir, "models") create_directory(target_root, False) for model in model_list: download_and_convert_model(target_root, model, force)
def save_model_metadata(model, current_date_time): create_directory('models/{}/models_{}/metadata'.format( object_type, current_date_time), format=False) metadata = dict() metadata['project'] = dict() metadata['project']['type'] = os.getenv('PROJECT_TYPE').lower() metadata['project']['object_type'] = object_type metadata['project']['date_time'] = current_date_time metadata['models'] = dict() model_files = get_subfiles('models/{}/models_{}'.format( object_type, current_date_time)) for model_name in model_files: epoch = int(model_name.split('.')[1].split('_')[1].split('-')[0]) metadata['models'][str(epoch)] = model_name # save model config in json format model_config_path = save_model_config(model, current_date_time) metadata['config'] = dict() metadata['config']['model'] = model_config_path metadata['config']['data_preparation'] = dict() metadata['config']['data_preparation']['source_data'] = os.getenv( 'SATELLITE_IMAGE_PATH') metadata['config']['data_preparation']['tilesize'] = int( os.getenv('TILESIZE')) metadata['config']['data_preparation']['step'] = int(os.getenv('STEP')) metadata['config']['data_preparation']['width'] = int(os.getenv('WIDTH')) metadata['config']['data_preparation']['height'] = int(os.getenv('HEIGHT')) metadata['config']['data_preparation']['percent_image_with_no_annotations'] = \ float(os.getenv('PERCENT_IMAGE_WITH_NO_ANNOTATIONS')) metadata['config']['data_preparation']['min_annotations_per_image'] = int( os.getenv('MIN_ANNOTATIONS_PER_IMAGE')) metadata['config']['data_preprocessing'] = dict() metadata['config']['data_preprocessing']['percent_valid'] = float( os.getenv('PERCENTVALID')) metadata['config']['data_preprocessing']['percent_test'] = float( os.getenv('PERCENTTEST')) metadata['config']['train'] = dict() metadata['config']['train']['epochs'] = int(os.getenv('EPOCHS')) metadata['config']['train']['train_datasetsize'] = int( os.getenv('TRAIN_DATASETSIZE')) metadata['config']['train']['valid_datasetsize'] = int( os.getenv('VALID_DATASETSIZE')) metadata['config']['train']['batchsize'] = int(os.getenv('BATCHSIZE')) with open( 'models/{}/models_{}/metadata/metadata_{}.json'.format( object_type, current_date_time, current_date_time), 'w') as f: json.dump(metadata, f)
def convert_config(db_session, http_request, template, schedule_form): config_form = init_config_form(db_session, http_request, get=True) success, err_msg = download_latest_config_migration_tool() if not success: return render_template(template, config_form=config_form, input_filename="", err_msg=err_msg, schedule_form=schedule_form, install_action=get_install_migrations_dict(), server_time=datetime.datetime.utcnow()) # check if the post request has the file part if 'file' not in http_request.files: flash('No file in request.') return render_template( template, config_form=config_form, input_filename="", err_msg="Internal error - No file.", schedule_form=schedule_form, install_action=get_install_migrations_dict(), server_time=datetime.datetime.utcnow(), ) input_file = http_request.files['file'] # if user does not select file, browser also # submit a empty part without filename if input_file.filename == '': flash('No selected file.') return render_template( template, config_form=config_form, input_filename="", err_msg="Internal error - No selected file.", schedule_form=schedule_form, install_action=get_install_migrations_dict(), server_time=datetime.datetime.utcnow(), ) config_conversion_path = get_config_conversion_path() create_directory(config_conversion_path) if input_file: filename = secure_filename(input_file.filename) input_file.save(os.path.join(config_conversion_path, filename)) return render_template(template, config_form=config_form, input_filename=input_file.filename, err_msg="", schedule_form=schedule_form, install_action=get_install_migrations_dict(), server_time=datetime.datetime.utcnow())
def download_loop(clear, verbose): create_directory("pdf", clear) INICIO = dt.date(2020, 4, 14) numero_dias = (INICIO.today() - INICIO).days for n in range(numero_dias + 1): data = INICIO + dt.timedelta(n) msg = download_if_inexistent(data) if verbose: print(msg)
def make_database_views(input_dir, output_dir): print("--- STEP 3: CREATE DATABASE VIEWS") scripts_output_dir = output_dir + "scripts/" create_directory(scripts_output_dir) make_lea_views(input_dir, scripts_output_dir) make_schools_views(input_dir, scripts_output_dir) print(f" * Database Views Complete ")
def init(): # Create the necessary supporting directories create_directory(get_log_directory()) create_directory(get_repository_directory()) create_directory(get_temp_directory()) create_directory(get_migration_directory()) create_directory(get_doc_central_directory()) if not is_ldap_supported(): print('LDAP authentication is not supported because it has not been installed.')
def init(): # Create the necessary supporting directories create_directory(get_log_directory()) create_directory(get_repository_directory()) create_directory(get_temp_directory()) create_directory(get_migration_directory()) create_directory(get_doc_central_directory()) if not is_ldap_supported(): print( 'LDAP authentication is not supported because it has not been installed.' )
def main(_file='', _data={}) -> int: if (_file): _data = read_data(_file) if check_file(_file) else {} if not _data: print_invalid_file(_file) return 0 COURSE_NAME = _data['course'] EVALUATION = _data['evaluation'] OCR = _data['ocr'] COPIES = _data['copies'] if _data.get('copies') else 1 create_directory(f'{os.getcwd()}/ANSWER_SHEETS/') if not os.path.exists( f'{os.getcwd()}/ANSWER_SHEETS/') else None ANSWER_SHEETS_DIR_PATH = f'{os.getcwd()}/ANSWER_SHEETS/{COURSE_NAME}_{EVALUATION}' cool_print( f'\nAnswer sheets will be available in: {ANSWER_SHEETS_DIR_PATH}', style='result') _current_time = asctime(localtime(time())) cool_print(f'\n\nInitializing program...[{_current_time}]', style='info') if png_template_exists(f'{TEMPLATE_DIRECTORY}/{_data["template"]}'): cool_print_decoration( 'ERROR: Found template in .pdf format.\nConverting template to .png format...', 'danger') pdf_to_png(f'{TEMPLATE_DIRECTORY}/{_data["template"]}.pdf') create_directory(ANSWER_SHEETS_DIR_PATH ) if not os.path.exists(ANSWER_SHEETS_DIR_PATH) else None _data['template'] = f'{TEMPLATE_DIRECTORY}/{_data["template"]}.png' cool_print(f'\nPreparing data...', style='info') _files = make_files(_data, _save_file=False) cool_print(f'\nAdding texts to templates...', style='info') _answer_sheets = composite_multiple_images( _files, _save_pages=False, _save_path=ANSWER_SHEETS_DIR_PATH, ocr=OCR) _answer_sheets = _answer_sheets if COPIES == 1 else make_copies( _answer_sheets, COPIES) cool_print(f'\nMerging files...[could take a while]', style='info') pdf_merger(_answer_sheets, f'{ANSWER_SHEETS_DIR_PATH}/compilation.pdf') cool_print(f'\nDone!', style='info') return 1
def convert_loop(clear, verbose): create_directory("txt", clear) lista_pdfs = sorted(glob("./pdf/*.pdf"), key=getmtime) for arquivo in lista_pdfs: txt_path = arquivo.replace("pdf", "txt", 2) if exists(txt_path): if verbose: print(f"{txt_path} já existe") else: comando = f"pdftotext {arquivo} {txt_path}" if verbose: print(f"Convertendo {arquivo}") os.system(comando)
def load_skin(self, skin_name, system_skin_dir=None, user_skin_dir=None): '''Load skin, return True if load finish, otherwise return False.''' try: # Save skin dir. self.skin_name = skin_name if system_skin_dir: self.system_skin_dir = system_skin_dir create_directory(self.system_skin_dir) if user_skin_dir: self.user_skin_dir = user_skin_dir create_directory(self.user_skin_dir) self.skin_dir = self.get_skin_dir() # Load config file. self.config = Config(self.get_skin_file_path("config.ini")) self.config.load() # Get theme config. self.theme_name = self.config.get("theme", "theme_name") # Get application config. self.app_id = self.config.get("application", "app_id") self.app_version = self.config.getfloat("application", "app_version") # Get background config. self.image = self.config.get("background", "image") self.x = self.config.getfloat("background", "x") self.y = self.config.getfloat("background", "y") self.scale_x = self.config.getfloat("background", "scale_x") self.scale_y = self.config.getfloat("background", "scale_y") self.dominant_color = self.config.get("background", "dominant_color") # Get action config. self.deletable = self.config.getboolean("action", "deletable") self.editable = self.config.getboolean("action", "editable") self.vertical_mirror = self.config.getboolean("action", "vertical_mirror") self.horizontal_mirror = self.config.getboolean("action", "horizontal_mirror") # Generate background pixbuf. self.background_pixbuf = gtk.gdk.pixbuf_new_from_file(self.get_skin_file_path(self.image)) # Save skin name. self.save_skin_name() return True except Exception, e: print "load_skin error: %s" % (e) return False
def __init__(self, system_theme_dir, user_theme_dir): '''Init theme.''' # Init. self.system_theme_dir = system_theme_dir self.user_theme_dir = user_theme_dir self.theme_info_file = "theme.txt" self.ticker = 0 self.pixbuf_dict = {} self.color_dict = {} self.alpha_color_dict = {} self.shadow_color_dict = {} # Create directory if necessarily. for theme_dir in [self.system_theme_dir, self.user_theme_dir]: create_directory(theme_dir)
def __init__(self, **kwargs): ReportWriter.__init__(self, **kwargs) self.user = kwargs.pop('user') self.smu_loader = kwargs.pop('smu_loader') self.smu_list = kwargs.pop('smu_list') self.sp_list = kwargs.pop('sp_list') self.smu_list = sorted(self.smu_list, key=lambda x: x.posted_date, reverse=True) self.sp_list = sorted(self.sp_list, key=lambda x: x.posted_date, reverse=True) temp_user_dir = create_temp_user_directory(self.user.username) self.output_file_directory = os.path.normpath(os.path.join(temp_user_dir, 'software_information_export')) create_directory(self.output_file_directory) make_file_writable(self.output_file_directory)
def __init__(self, **kwargs): ReportWriter.__init__(self, **kwargs) self.user = kwargs.pop('user') self.region_name = kwargs.pop('region_name') self.chassis_summary_iter = kwargs.pop('chassis_summary_iter') self.model_name_summary_iter = kwargs.pop('model_name_summary_iter') self.inventory_without_serial_number_iter = kwargs.pop('inventory_without_serial_number_iter') self.inventory_with_duplicate_serial_number_iter = kwargs.pop('inventory_with_duplicate_serial_number_iter') temp_user_dir = create_temp_user_directory(self.user.username) self.output_file_directory = os.path.normpath(os.path.join(temp_user_dir, 'inventory_information_export')) create_directory(self.output_file_directory) make_file_writable(self.output_file_directory)
def deconstruct(self): """Deconstruct an entire combined image file into the individual blocks.""" # Do some sanity checks on input. if not os.path.exists(self.img_path): self.error_handler("Input file does not exist: %s" % (self.img_path)) self.source = Image.open(self.img_path) (source_width, source_height) = self.source.size if (source_width % self.grid_width != 0) or (source_height % self.grid_height != 0): self.error_handler("Invalid image size.") self.block_size = source_width / self.grid_width if self.block_size != (source_height / self.grid_height): self.error_handler("Blocks not square.") res = utils.create_directory(self.workdir) if res != True: self.error_handler(res) # Loop over everything, ignoring blank spaces. for i in xrange(self.grid_width): for j in xrange(self.grid_height): if (i, j) in self.map: block = self.map[(i, j)] self.write_block_to_file(i, j, block)
def transform_data(edges_file, edges_tuples, output_dir="", several_out_files=False, undirected=False): ''' Loads the data from a provided file and transforms the data to a two column dataframe with source and target nodes (edge_index/hyperedge_index). Finally saves data to csv file/files. Args: edges_file: filepath to a csv file where nodes and hyperrelations are columns and traversed graph paths are rows edge_tuples: list of tuples: (node, containing hyperrelation) or (hyperrelation, containing hyperrelation) output_dir: directory where the output dataframe should be saved several_out_files: if True saves transformed dataframes separately for each tuple from 'edge_tuples' (default: False) undirected: if True creates an edge_index file/files for undirected graph ''' utils.create_directory(output_dir) edges_df = pd.read_csv(edges_file) for column in edges_df.columns: edges_df[column] = edges_df[column].str.replace("V", "") temp_cols = ["source", "target"] if several_out_files == True: for edge_tuple in edges_tuples: temp_df = pd.DataFrame(columns=temp_cols) file_out_name = "{}{}_{}_edges.csv".format(output_dir, edge_tuple[0], edge_tuple[1]) temp_df[temp_cols] = edges_df[edge_tuple] if undirected: temp_df = make_undirected(temp_df) temp_df.to_csv(file_out_name, index=False) else: temp_df = pd.DataFrame(columns=temp_cols) for edge_tuple in edges_tuples: temp_df = pd.concat([ temp_df, edges_df[edge_tuple].rename(columns={ edge_tuple[0]: temp_cols[0], edge_tuple[1]: temp_cols[1] }) ], ignore_index=True) temp_df = temp_df.drop_duplicates() if undirected: temp_df = make_undirected(temp_df) temp_df.to_csv(output_dir + "all_edges_index.csv", index=False)
def setup(): # returns None # create directory if needed for dir in const.BASIC_DIR_TREE: if utils.check_if_directory_exists(dir) == False: utils.create_directory(dir) # create files if needed if utils.check_if_file_exists(const.CONFIG_FILE) == False: # utils.create_file(const.CONFIG_FILE) do_file_from_list(const.CONFIG_FILE, const.DEFAULT_CONFIG_FILE) if utils.check_if_file_exists(const.KEYS_DOWN_FILE) == False: utils.create_file(const.KEYS_DOWN_FILE) do_file_from_list(const.KEYS_DOWN_FILE, const.DEFAULT_KEYS_DOWN_FILE) if utils.check_if_file_exists(const.KEYS_UP_FILE) == False: utils.create_file(const.KEYS_UP_FILE) do_file_from_list(const.KEYS_UP_FILE, const.DEFAULT_KEYS_UP_FILE) if utils.check_if_file_exists(const.MODS_FILE) == False: utils.create_file(const.MODS_FILE) do_file_from_list(const.MODS_FILE, const.DEFAULT_MODS_FILE)
def download_from_object_storage(config, directory, bucket_name, object_name_list): object_storage = oci.object_storage.ObjectStorageClient(config) namespace = object_storage.get_namespace().data # verify local folders to create object_folder_list = set(o.rsplit("/", 1)[0] for o in object_name_list) for object_folder in object_folder_list: utils.create_directory(os.path.join(directory, object_folder)) for object_name in object_name_list: print('Downloading object: {}'.format(object_name)) get_obj = object_storage.get_object(namespace, bucket_name, object_name) with open(os.path.join(directory, object_name), 'wb') as f: for chunk in get_obj.data.raw.stream(1024 * 1024, decode_content=False): f.write(chunk)
def __init__(self, **kwargs): ReportWriter.__init__(self, **kwargs) self.user = kwargs.pop('user') self.serial_number = kwargs.pop('serial_number') self.region_names = kwargs.pop('region_names') self.chassis_types = kwargs.pop('chassis_types') self.software_versions = kwargs.pop('software_versions') self.model_names = kwargs.pop('model_names') self.partial_model_names = kwargs.pop('partial_model_names') self.vid = kwargs.pop('vid') self.available_inventory_iter = kwargs.pop('available_inventory_iter') self.in_use_inventory_iter = kwargs.pop('in_use_inventory_iter') temp_user_dir = create_temp_user_directory(self.user.username) self.output_file_directory = os.path.normpath(os.path.join(temp_user_dir, 'inventory_information_export')) create_directory(self.output_file_directory) make_file_writable(self.output_file_directory)
def export_command_profiles(): db_session = DBSession() profiles_list = request.args.getlist('profiles_list[]')[0].split(",") db_profiles = db_session.query(CustomCommandProfile).all() d = {"CSM Server:Custom Command Profile": {}} for profile in db_profiles: if profile.profile_name in profiles_list: d["CSM Server:Custom Command Profile"][profile.profile_name] = profile.command_list temp_user_dir = create_temp_user_directory(current_user.username) custom_command_export_temp_path = os.path.normpath(os.path.join(temp_user_dir, "custom_command_export")) create_directory(custom_command_export_temp_path) make_file_writable(custom_command_export_temp_path) with open(os.path.join(custom_command_export_temp_path, 'custom_command_profiles.json'), 'w') as command_export_file: command_export_file.write(json.dumps(d, indent=2)) return send_file(os.path.join(custom_command_export_temp_path, 'custom_command_profiles.json'), as_attachment=True)
def download_session_logs(file_list): if hasattr(current_user, 'username'): username = current_user.username else: username = g.api_user.username temp_user_dir = create_temp_user_directory(username) session_zip_path = os.path.normpath(os.path.join(temp_user_dir, "session_logs")) zip_file = os.path.join(session_zip_path, "session_logs.zip") create_directory(session_zip_path) make_file_writable(session_zip_path) zout = zipfile.ZipFile(zip_file, mode='w') for f in file_list: zout.write(os.path.normpath(f), os.path.basename(f)) zout.close() return send_file(zip_file, as_attachment=True)
def write(self, dpath, use_text=True, use_tags=True, overwrite=False): r""" The patient XML tree is written to a file. Only the directory path needs to be specified, since the filename is already defined. The <TEXT> and <TAGS> fields can be omitted. Parameters ---------- dpath : str The output directory path. If necessary, directories are created. The filename is the same (as defined). use_text : bool, optional If `True` the generated XML contains the <TEXT> field, otherwise the <TEXT> field is omitted. Default: `True`. use_tags : bool, optional If `True` the generated XML contains the <TAGS> field, otherwise the <TAGS> field is omitted. Default: `True`. overwrite: bool, optional If `True` overwritten is allowed, otherwise it is not. Default: `False`. Raises ------ AssertionError The output directory path cannot be the same as the input directory path. """ dpath = os.path.abspath(dpath) # create directory (if necessary) create_directory(dpath) e = 'Writing to the same directory is not allowed!' assert not os.path.samefile(self._dpath, dpath), e fpath = os.path.join(dpath, self._fname) if os.path.exists(fpath): e = 'Filepath already exists and it is not a file!' assert os.path.isfile(fpath), e assert overwrite, 'Overwitten is not allowed!' with open(fpath, mode='w', encoding='utf-8') as f: _ = f.write(self.to_xml(use_text=use_text, use_tags=use_tags))
def convert_config(db_session, http_request, template, schedule_form): config_form = init_config_form(db_session, http_request, get=True) success, err_msg = download_latest_config_migration_tool() if not success: return render_template(template, config_form=config_form, input_filename="", err_msg=err_msg, schedule_form=schedule_form, install_action=get_install_migrations_dict(), server_time=datetime.datetime.utcnow()) # check if the post request has the file part if 'file' not in http_request.files: flash('No file in request.') return render_template(template, config_form=config_form, input_filename="", err_msg="Internal error - No file.", schedule_form=schedule_form, install_action=get_install_migrations_dict(), server_time=datetime.datetime.utcnow(),) input_file = http_request.files['file'] # if user does not select file, browser also # submit a empty part without filename if input_file.filename == '': flash('No selected file.') return render_template(template, config_form=config_form, input_filename="", err_msg="Internal error - No selected file.", schedule_form=schedule_form, install_action=get_install_migrations_dict(), server_time=datetime.datetime.utcnow(),) config_conversion_path = get_config_conversion_path() create_directory(config_conversion_path) if input_file: filename = secure_filename(input_file.filename) input_file.save(os.path.join(config_conversion_path, filename)) return render_template(template, config_form=config_form, input_filename=input_file.filename, err_msg="", schedule_form=schedule_form, install_action=get_install_migrations_dict(), server_time=datetime.datetime.utcnow())
def write(self, dpath, use_text=True, use_tags=True, overwrite=False): r""" The patient XML files are written into a specific directory. Only the directory path needs to be specified, since the patient filenames are already defined. The <TEXT> and <TAGS> fields can be omitted. Parameters ---------- dpath : str The output directory path. If necessary, directories are created. use_text : bool, optional If `True` the generated XML contains the <TEXT> field, otherwise the <TEXT> field is omitted. Default: `True`. use_tags : bool, optional If `True` the generated XML contains the <TAGS> field, otherwise the <TAGS> field is omitted. Default: `True`. overwrite: bool, optional If `True` overwritten is allowed, otherwise it is not. Default: `False`. Raises ------ AssertionError The output directory path cannot be the same as the input directory path. """ dpath = os.path.abspath(dpath) # create directory (if necessary) create_directory(dpath) e = 'Writing to the same directory is not allowed!' assert not os.path.samefile(self._dpath, dpath), e for p in self._patients: p.write( dpath=dpath, use_text=use_text, use_tags=use_tags, overwrite=overwrite, )
def parse_pdf2img(filename, folder_img): try: with tempfile.TemporaryDirectory() as tmppath: images = convert_from_path(filename, dpi=80, fmt='jpeg', strict=False, last_page=10, output_folder=tmppath) utils.create_directory(folder_img) files = os.listdir(tmppath) for file in files: src = os.path.join(tmppath, file) shutil.move(src, folder_img) return True except: logger.error(("pdf2image could not convert " + " the document '{}'").format(filename)) return False
def get_predictions(): create_directory(os.getenv('PREDICTION_IMAGES_PATH')) # get all test files test_filenames = get_subfiles(os.getenv('TEST_IMAGES_PATH')) # test_filenames = test_filenames[:1000] # get test data generator testgen = test_generator(os.getenv('TEST_IMAGES_PATH')) test_batch_size = len(test_filenames) # start testing model = get_model() print('\nStarting testing ...') print('Using model - {}'.format(weight_file_path)) results = model.predict_generator(testgen, test_batch_size, verbose=1) print('DONE !') # save predictions save_predictions(results, test_filenames)
def download_session_logs(file_list): if hasattr(current_user, 'username'): username = current_user.username else: username = g.api_user.username temp_user_dir = create_temp_user_directory(username) session_zip_path = os.path.normpath( os.path.join(temp_user_dir, "session_logs")) zip_file = os.path.join(session_zip_path, "session_logs.zip") create_directory(session_zip_path) make_file_writable(session_zip_path) zout = zipfile.ZipFile(zip_file, mode='w') for f in file_list: zout.write(os.path.normpath(f), os.path.basename(f)) zout.close() return send_file(zip_file, as_attachment=True)
def save_predictions(results, test_filenames): # save predictions - images and masks print('\nSaving test results') save_result(os.getenv('PREDICTION_IMAGES_PATH'), results, test_filenames, flag_multi_class=False, num_class=2) print('DONE !') # save image and predictions combined if os.getenv('SAVE_COMBINED') == 'TRUE': create_directory(os.getenv('COMBINED_IMAGES_PATH')) imagesdir = os.getenv('TEST_IMAGES_PATH') masksdir = os.getenv('PREDICTION_IMAGES_PATH') suffix = '_predict' print('\nSaving test results - images and masks combined') save_images_and_masks(imagesdir, masksdir, suffix, save=True) print('DONE !')
def encode_ids(edge_csv_file_path, nodes_labs_file_path, output_path): ''' Encodes nodes or hyperrelations ids so that they are within range 1...N where N is number of nodes in nodes_labels. Saves data to csv in provided directory. Args: edge_csv_file_path: filepath to a csv file where nodes and hyperrelations are columns and traversed graph paths are rows nodes_labs_file_path: filepath to a two columns csv files with nodes ids and their labels output_path: file directory to save encoded dataframes ''' utils.create_directory(output_path) edge_index_df = pd.read_csv(edge_csv_file_path) nodes_labels = pd.read_csv(nodes_labs_file_path) lbl_encoder = LabelEncoder() lbl_encoder.fit(nodes_labels["node"]) nodes_labels["node"] = lbl_encoder.transform(nodes_labels["node"]) edge_index_df["source"] = lbl_encoder.transform(edge_index_df["source"]) edge_index_df["target"] = lbl_encoder.transform(edge_index_df["target"]) edge_index_df.to_csv("{}all_edges_index.csv".format(output_path), index=False) nodes_labels.to_csv("{}nodes_labels.csv".format(output_path), index=False)
def export_software_profiles(): db_session = DBSession() software_profile_list = request.args.getlist('software_profile_list[]')[0].split(",") software_profiles = get_software_profile_list(db_session) d = {"CSM Server:Software Profile": {}} for software_profile in software_profiles: if software_profile.name in software_profile_list: d["CSM Server:Software Profile"][software_profile.name] = software_profile.packages temp_user_dir = create_temp_user_directory(current_user.username) software_profile_export_temp_path = os.path.normpath(os.path.join(temp_user_dir, "software_profile_export")) create_directory(software_profile_export_temp_path) make_file_writable(software_profile_export_temp_path) with open(os.path.join(software_profile_export_temp_path, 'software_profiles.json'), 'w') as command_export_file: command_export_file.write(json.dumps(d, indent=2)) return send_file(os.path.join(software_profile_export_temp_path, 'software_profiles.json'), as_attachment=True)
def setup_data(): pretty_print("SETUP DATA") extract_directory = f"{INPUT_DIR}extracts/" """ See if needed files currently exist in input directory. If not, see if extract file already exists correctly If not, retrieve and extract accordingly Move extracted files to correct directory and simplified filename Remove extra directory and files """ if (needed_files_exists()): pretty_print("Needed Files Already Exist", True) else: if (extracted_files_exists(extract_directory)): pretty_print("Extract Already Exist", True) else: create_directory(extract_directory) pretty_print("Fetching CRDC Data From Public Website (34MB)", True) zip_file_name = get_filename_from_url(CRDC_DATA_URL) zip_file_name = fetch_file(CRDC_DATA_URL, extract_directory, zip_file_name) pretty_print("Extracting Zip At ", True, extract_directory + zip_file_name) unzip(extract_directory + zip_file_name, extract_directory) pretty_print("Moving Files In Place", True) formatted_files_list = list( map( lambda x: { "src_path": x["extracted_path"], "dest_path": x["needed_file_name"] }, CRDC_FILES)) rename_files(formatted_files_list, extract_directory, INPUT_DIR) pretty_print("Cleaning Up", True) remove_directory(extract_directory) # create_directory(OUTPUT_DIR, True) create_directory(MIGRATION_DIR, True)
def __init__(self, system_theme_dir, user_theme_dir): ''' Initialize Theme class. @param system_theme_dir: Default theme directory. @param user_theme_dir: User's theme save directory, generic is ~/.config/project-name/theme ''' # Init. self.system_theme_dir = system_theme_dir self.user_theme_dir = user_theme_dir self.theme_info_file = "theme.txt" self.ticker = 0 self.pixbuf_dict = {} self.color_dict = {} self.alpha_color_dict = {} self.shadow_color_dict = {} # Create directory if necessarily. for theme_dir in [self.system_theme_dir, self.user_theme_dir]: create_directory(theme_dir)
def setup_data(input_dir="./input/"): print("--- STEP 1: SETUP DATA") extract_directory = f"{input_dir}extracts/" """ See if needed files currently exist in input directory. If not, see if extract file already exists correctly If not, retrieve and extract accordingly Move extracted files to correct directory and simplified filename Remove extra directory and files """ if (needed_files_exists(input_dir)): print(" * Needed Files Already Exist") else: if (extracted_files_exists(extract_directory)): print(" * Extract Already Exist") else: create_directory(extract_directory) print(" * Fetching CRDC Data From Public Website (34MB)") zip_file_name = get_filename_from_url(CRDC_DATA_URL) zip_file_name = fetch_file(CRDC_DATA_URL, extract_directory, zip_file_name) print(" * Extracting Zip At ", extract_directory + zip_file_name) unzip(extract_directory + zip_file_name, extract_directory) print(" * Moving Files In Place") formatted_files_list = list( map( lambda x: { "src_path": x["extracted_path"], "dest_path": x["needed_file_name"] }, CRDC_FILES)) rename_files(formatted_files_list, extract_directory, input_dir) print(" * Cleaning Up") remove_directory(extract_directory) print(" * Setup Complete")
def optimise_parameters(): pyramid_levels = [3, 4, 5] # pyramid_levels = [x for x in range(3,6)] rotations = [[x for x in range(0, 360, rot)] for rot in range(20, 35, 5)] gaussian_parameters = [[5, 5, 15]] # Create results directory utils.create_directory(config.RESULTS_DIR) for level in pyramid_levels: for rots in rotations: for g_n, gaussian in enumerate(gaussian_parameters): step_size = rots[1] - rots[0] row, col, dev = gaussian g = utils.gaussian_kernel(row, col, dev) utils.delete_directory(config.TEMPLATE_OUTPUT_DIR) print( 'training rotation {} level {} gaussian {}-{}-{}'.format( step_size, level, row, col, dev), rots, level) start = time.time() template_matching(config.TRAINING_DIR, config.TEMPLATE_OUTPUT_DIR, level, rots, g) new_dir = config.RESULTS_DIR + 'level{}-rot{}-g-{}-{}-{}/'.format( level, step_size, row, col, dev) utils.create_directory(new_dir) print('testing', rots, level) images = test_template_matching(config.TESTING_DIR, config.TEMPLATE_OUTPUT_DIR) end = time.time() time_elapsed = end - start utils.write_to_file(new_dir + 'time.txt', time_elapsed) for idx, im in enumerate(images): cv2.imwrite(new_dir + '{}.png'.format(idx), im) return True
def template_matching(data_dir, template_dir, pyramid_depth=4, rotations=None, gaussian=None): if rotations is None: rotations = [x for x in range(0, 360, 30)] if gaussian is None: gaussian = utils.gaussian_kernel(5, 5, 15) image_names = utils.get_files(data_dir, extension='.png') for image_name in image_names: image = cv2.imread(data_dir + image_name + '.png', cv2.IMREAD_UNCHANGED) image_filtered = tm.pre_process_image(image) # Create a Gaussian pyramid of given depth for each image pyramid = tm.create_gaussian_pyramid(image_filtered, gaussian, pyramid_depth) # Create directory for class image_class = tm.get_class_name(image_name) class_dir = template_dir + image_class + '/' utils.create_directory(class_dir) # Path(class_dir).mkdir(parents=True, exist_ok=True) # Rotate each scaled image for scale_index, scaled_image in enumerate(pyramid): for angle in rotations: rotated_image = tm.rotate_image(scaled_image, angle) # Save image to png file file_name = image_class + "-level" + str( scale_index) + "-rotation" + str(angle) + ".png" cv2.imwrite(class_dir + file_name, rotated_image) return True
def __init__(self, **kwargs): super(ConformanceReportWriter, self).__init__(**kwargs) self.style_title = xlwt.easyxf('font: height 350, bold on; align: vert centre, horiz center;') self.style_bold = xlwt.easyxf('font: bold on, height 260;') self.style_summary = xlwt.easyxf('font: height 220;') self.style_center = xlwt.easyxf('align: vert centre, horiz center;') self.user = kwargs.pop('user') self.conformance_report = kwargs.pop('conformance_report') self.locale_datetime = kwargs.pop('locale_datetime') self.include_host_packages = kwargs.pop('include_host_packages') self.exclude_conforming_hosts = kwargs.pop('exclude_conforming_hosts') self.wb = xlwt.Workbook() self.ws = self.wb.add_sheet('Conformance Report') self.ws.set_portrait(False) temp_user_dir = create_temp_user_directory(self.user.username) self.output_file_directory = os.path.normpath(os.path.join(temp_user_dir, "conformance_report")) create_directory(self.output_file_directory) make_file_writable(self.output_file_directory) self.row = 0
def reconstruct(self, res=[16]): """Assemble the grid from the individual block files. If more than one resolution is desired, the largest resolution is used and the end result is scaled down. The output files will have the size denoted in the filename.""" res = list(set(res)) # Squish duplicates. if len(res) == 0: self.error_handler("No resolutions selected.") res.sort(reverse=True) resp = utils.create_directory(self.outdir) if resp != True: utils.raise_error(resp) self.block_size = res.pop(0) self.output = Image.new( "RGBA", (self.block_size * self.grid_width, self.block_size * self.grid_height), (0, 0, 0, 0) ) # Loop over everything, ignoring empty spaces. for i in xrange(self.grid_width): for j in xrange(self.grid_height): if (i, j) in self.map: block = self.map[(i, j)] self.read_and_paste_block(i, j, block) # Save the primary output size. name_base = os.path.split(self.img_path)[1] ext = "_" + str(self.block_size) + "x" + str(self.block_size) + ".png" outfile = self.outdir + os.sep + name_base.replace(".png", ext) self.output.save(outfile) # Save additional output sizes. for size in res: ext = "_" + str(size) + "x" + str(size) + ".png" outfile = self.outdir + os.sep + name_base.replace(".png", ext) dims = (size * self.grid_width, size * self.grid_height) img = self.output.resize(dims, Image.BICUBIC) img.save(outfile)
parser.add_argument('--fixed-embeddings',\ help='Loads the corresponding embeddings from the given word embedding file') parser.add_argument('--learn-embeddings',\ help='Loads the corresponding embeddings from that only exists in the test sentence') if __name__=="__main__": args = parser.parse_args() md = Metadata(args, args.filename, args.fixed_embeddings or args.learn_embeddings) varlist = list(map(str, [os.path.basename(args.filename), os.path.basename(args.validation_filename), \ args.iterations, args.hidden, args.l2, args.dropout_rare, args.dropout,\ args.fixed_embeddings is not None, args.learn_embeddings is not None])) directory_model = 'Model_' + '_'.join(varlist) utils.create_directory(directory_model) if args.load_reader: print('... loading reader') with open(os.path.join(directory_model, 'reader.pkl'), 'rb') as f: reader = pickle.load(f) else: print('... Generating new reader') reader = Reader(md, minimum_occurrence=2) #reader.save(directory_model) # Generate the training set num_sentences = len(reader.sentences) num_words = len(reader.word_dict) num_tags = len(reader.tag_dict)
def get_config_conversion_path(): temp_user_dir = create_temp_user_directory(current_user.username) config_conversion_path = os.path.normpath(os.path.join(temp_user_dir, "config_conversion")) create_directory(config_conversion_path) return config_conversion_path
# By importing models here, it forces creation of tables in the database for a new installation. # This will prevent gunicorn workers from trying to create the database tables all at the same time. # See csmserver launch script import models from utils import create_directory from constants import get_autlogs_directory, get_repository_directory, get_temp_directory # Create the necessary supporting directories create_directory(get_autlogs_directory()) create_directory(get_repository_directory()) create_directory(get_temp_directory())
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. # ============================================================================= from models import initialize from models import SystemVersion from sqlalchemy import inspect from database import DBSession, CURRENT_SCHEMA_VERSION, engine from utils import create_directory, is_ldap_supported from constants import get_log_directory, get_repository_directory, get_temp_directory from schema.loader import get_schema_migrate_class import traceback # Create the necessary supporting directories create_directory(get_log_directory()) create_directory(get_repository_directory()) create_directory(get_temp_directory()) def init(): if not is_ldap_supported(): print('LDAP authentication is not supported because it has not been installed.') db_session = DBSession() system_version = SystemVersion.get(db_session) # Handles database schema migration starting from the next schema version for version in range(system_version.schema_version + 1, CURRENT_SCHEMA_VERSION + 1): handler_class = get_schema_migrate_class(version) if handler_class is not None: