def save(self): """save to campaign_dir/data/YYYYMM/state_YYMMDD_HHMMSS.json and campaign_dir/data/YYYYMM/state_full_YYMMDD_HHMMSS.json""" if not self.goal_results or not self.article_results: raise RuntimeError( 'only intended to be called after a full results population with from_api()' ) save_timestamp = datetime.datetime.utcnow().isoformat() result_path = self.campaign.base_path + self.timestamp.strftime( STATE_PATH_TMPL) mkdir_p(os.path.split(result_path)[0]) result_data = { 'campaign_name': self.campaign.name, 'timestamp': self.timestamp, 'save_date': save_timestamp, 'campaign_results': self.campaign_results, 'goal_results': self.goal_results, 'title_list': self.campaign.article_title_list } with atomic_save(result_path) as f: json.dump(result_data, f, indent=2, sort_keys=True, default=str) full_result_fn = self.timestamp.strftime(STATE_FULL_PATH_TMPL) full_result_path = self.campaign.base_path + full_result_fn result_data['article_results'] = [ attr.asdict(a) for a in self.article_list ] with atomic_save(full_result_path) as f: gzf = gzip.GzipFile(filename=full_result_fn, fileobj=f) json.dump(result_data, gzf, default=str) gzf.close() return
def update_index(pypier_path): new_index_bytes, new_readme_bytes = generate_index(pypier_path) index_path = pypier_path + '/packages/index.html' readme_path = pypier_path + '/packages/README.md' with fileutils.atomic_save(index_path) as f: f.write(new_index_bytes) with fileutils.atomic_save(readme_path) as f: f.write(new_readme_bytes) return
def _main(): start_time = time.time() with open('projects.yaml') as f: projects = yaml.load(f)['projects'] entries = [] for p in projects: print 'processing', p['name'] info = dict(p) if info.get('skip'): continue info['url'] = info.get('url', info.get('gh_url')) if info.get('gh_url'): gh_info = get_gh_project_info(info['gh_url']) info.update(gh_info) info['is_zerover'] = info.get('is_zerover', not info.get('emeritus', False)) entries.append(info) from pprint import pprint pprint(entries) res = {'projects': entries, 'gen_date': datetime.datetime.utcnow().isoformat(), 'gen_duration': time.time() - start_time} with atomic_save('projects.json') as f: f.write(json.dumps(res, indent=2, sort_keys=True, default=_json_default)) return
def write_yaml(self, stat: OutputProduct, tmp_filenames: List[Path], yaml_filename=None, multiband=True): output_filenames = [self.output_filename_tmpname[tmp_filename] for tmp_filename in tmp_filenames] uris = [output_filename.absolute().as_uri() for output_filename in output_filenames] def layer(index): if multiband: return index + 1 return 1 band_uris = {name: {'layer': layer(index), 'path': uris[index]} for index, name in enumerate(stat.product.measurements)} datasets = self._find_source_datasets(stat, uri=None, band_uris=band_uris) if yaml_filename is None: yaml_filename = str(output_filenames[0].with_suffix('.yaml')) # Write to Yaml if len(datasets) == 1: # I don't think there should ever be more than 1 dataset in here... _LOG.info('writing dataset yaml for %s to %s', stat, yaml_filename) with fileutils.atomic_save(yaml_filename) as yaml_dst: yaml_dst.write(datasets.values[0]) else: _LOG.error('Unexpected more than 1 dataset %r being written at once, ' 'investigate!', datasets)
def build_pathset(collection: Collection, cache_path: Path = None, log=_LOG) -> dawg.CompletionDAWG: """ Build a combined set (in dawg form) of all dataset paths in the given index and filesystem. Optionally use the given cache directory to cache repeated builds. """ locations_cache = cache_path.joinpath(query_name( collection.query), 'locations.dawg') if cache_path else None if locations_cache: fileutils.mkdir_p(str(locations_cache.parent)) log = log.bind(collection_name=collection.name) if locations_cache and not cache_is_too_old(locations_cache): path_set = dawg.CompletionDAWG() log.debug("paths.trie.cache.load", file=locations_cache) path_set.load(str(locations_cache)) else: log.info("paths.trie.build") path_set = dawg.CompletionDAWG( chain(collection.iter_index_uris(), collection.iter_fs_uris())) log.info("paths.trie.done") if locations_cache is not None: log.debug("paths.trie.cache.create", file=locations_cache) with fileutils.atomic_save(str(locations_cache)) as f: path_set.write(f) return path_set
def write_yaml(self, stat: OutputProduct, tmp_filename: Path): output_filename = self.output_filename_tmpname[tmp_filename] uri = output_filename.absolute().as_uri() def band_info(measurement_name): return { 'layer': list(stat.product.measurements).index(measurement_name) + 1, 'path': uri } band_uris = { name: band_info(name) for name in stat.product.measurements } datasets = self._find_source_datasets(stat, uri=uri, band_uris=band_uris) yaml_filename = str(output_filename.with_suffix('.yaml')) # Write to Yaml if len( datasets ) == 1: # I don't think there should ever be more than 1 dataset in here... _LOG.info('writing dataset yaml for %s to %s', stat, yaml_filename) with fileutils.atomic_save(yaml_filename) as yaml_dst: yaml_dst.write(datasets.values[0]) else: _LOG.error( 'Unexpected more than 1 dataset %r being written at once, ' 'investigate!', datasets)
def generate_project_daily(location, template_file, settings_file): """Generate a new file for each day of the project. Args: location (:obj:`str`): The location for the project structure. template_file (:obj:`str`): The template file to use for the generation. settings_file (:obj:`str`): The settings file to use for the markdown. Returns: int: Status code of success or failure. Anything except 0 is a failure. """ with io.open(settings_file) as fopen: data = yaml.safe_load(fopen.read()) project_vars = data['vars'] weeks = data['settings']['weeks'] days = 7 if data['settings']['days'] > 7 else data['settings']['days'] template_dir = os.path.dirname(template_file) jinja_env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True) template = jinja_env.get_template(os.path.basename(template_file)) for week in range(1, weeks + 1): LOGGER.info('Generating week %s files', week) for day in range(1, days + 1): day_path = os.path.join(location, 'week{:02d}'.format(week), 'day{:02d}'.format(day)) if _create_dir(day_path): return 1 readme_path = os.path.join(day_path, 'README.md') try: with fileutils.atomic_save(readme_path) as fopen: fopen.writelines(template.render(**project_vars).encode('utf-8')) except (OSError, IOError): LOGGER.exception('Failed to write data to %s', readme_path, exc_info=True) return 1 return 0
def fin(): mlog.debug('Closing database: {}'.format(db_file)) with open(cached_db_file, 'r') as ff: mlog.debug('Reverting to cached database: {}'.format(cached_db_file)) with atomic_save(db_file) as f: json.dump(json.load(ff), f) os.remove(cached_db_file)
def render(plist, pdir): "generate the list markdown from the yaml listing" topic_map = plist.get_projects_by_type('topic') topic_toc_text = format_tag_toc(topic_map) projects_by_topic = format_all_categories(topic_map) plat_map = plist.get_projects_by_type('platform') plat_toc_text = format_tag_toc(plat_map) projects_by_plat = format_all_categories(plat_map) context = { 'TOPIC_TOC': topic_toc_text, 'TOPIC_TEXT': projects_by_topic, 'PLATFORM_TOC': plat_toc_text, 'PLATFORM_TEXT': projects_by_plat, 'TOTAL_COUNT': len(plist.project_list) } templates_path = pdir + '/templates/' if not os.path.isdir(templates_path): raise APACLIError('expected "templates" directory at %r' % templates_path) for filename in iter_find_files(templates_path, '*.tmpl.md'): tmpl_text = open(filename).read() target_filename = os.path.split(filename)[1].replace('.tmpl', '') output_text = tmpl_text.format(**context) with atomic_save(pdir + '/' + target_filename) as f: f.write(output_text.encode('utf8')) return
def main(): plist = ProjectList.from_path('projects.yaml') print([p for p in plist.project_list if not p.desc]) topic_map = plist.get_projects_by_type('topic') topic_toc_text = format_tag_toc(topic_map) projects_by_topic = format_all_categories(topic_map) plat_map = plist.get_projects_by_type('platform') plat_toc_text = format_tag_toc(plat_map) projects_by_plat = format_all_categories(plat_map) context = { 'TOPIC_TOC': topic_toc_text, 'TOPIC_TEXT': projects_by_topic, 'PLATFORM_TOC': plat_toc_text, 'PLATFORM_TEXT': projects_by_plat, 'TOTAL_COUNT': len(plist.project_list) } for filename in iter_find_files(TEMPLATES_PATH, '*.tmpl.md'): tmpl_text = open(filename).read() target_filename = os.path.split(filename)[1].replace('.tmpl', '') output_text = tmpl_text.format(**context) with atomic_save(target_filename) as f: f.write(output_text.encode('utf8')) return
def render_home(ptcs): ctx = glom(ptcs, {'campaigns': [T.get_summary_ctx()]}) index_html = ASHES_ENV.render('index.html', ctx) index_path = STATIC_PATH + '/index.html' with atomic_save(index_path) as f: f.write(index_html.encode('utf-8')) return
def update(self, force=False, _act=None): "does it all" final_update_log_path = STATIC_PATH + 'campaigns/%s/update.log' % self.id _act['name'] = self.name _act['id'] = self.id _act['log_path'] = final_update_log_path now = datetime.datetime.utcnow() with atomic_save(final_update_log_path) as f: cur_update_sink = build_stream_sink(f) old_sinks = tlog.sinks tlog.set_sinks(old_sinks + [cur_update_sink]) try: self.load_article_list() self.load_latest_state() next_fetch = now if not self.latest_state else self.latest_state.timestamp + self.fetch_frequency if not force and next_fetch > now: tlog.critical('skip_fetch').success( '{cid} not out of date, skipping until next fetch at {next_fetch}. ', cid=self.id, next_fetch=next_fetch) return self.record_state() # defaults to now self.load_latest_state() self.prune_by_frequency() self.render_report() self.render_article_list() finally: tlog.set_sinks(old_sinks) return
def normalize(plist, pfile): """normalize project and tag order, checking for duplicates and format divergences, overwrites the yaml listing""" plist.normalize() new_yaml = plist.to_yaml() with atomic_save(pfile) as f: f.write(new_yaml.encode('utf8')) return
def save(self): if self._data is None: return with atomic_save(self._path) as outfp: self._data.prune() data = toml.dumps(self._data.to_dict()).encode("utf8") outfp.write(data)
def _pull_single_repo(proj, repo_dir, rm_cached=False): # TODO: turn rm_cached into a flag # TODO: shouldn't this be the callable submitted to the executor? vcs, url = proj.clone_info if url is None: print_err('project "%s" has unsupported vcs type for repo url: %r' % (proj.name, proj.repo_url)) return # TODO target_dir = repo_dir + proj.name_slug + '/' cwd = repo_dir mode = 'clone' if os.path.exists(target_dir): if not rm_cached: mode = 'update' cwd = target_dir else: # sanity check we're in the right place if os.path.exists(target_dir + '../.apatite_repo_dir'): shutil.rmtree(target_dir) else: raise Exception('non-apatite path blocking apatite clone: %r' % target_dir) try: cmd_tmpl = VCS_TMPLS[vcs][mode] except KeyError: raise Exception('unsupported operation %r with vcs %r' % (mode, vcs)) cmd = format_list(cmd_tmpl, cmd=vcs, url=url, target_dir=target_dir) # avoid asking for github username and password, since closing stdin doesn't work cur_dt = datetime.datetime.utcnow() started = time.time() proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env={'GIT_TERMINAL_PROMPT': '0'}) proc.project_name = proj.name_slug proc.target_dir = target_dir CUR_PROCS.append(proc) stdout, stderr = proc.communicate() CUR_PROCS.remove(proc) proc_res = ProcessResult( returncode=proc.returncode, stdout=stdout.decode('utf8'), stderr=stderr, start_time=started, end_time=datetime.datetime.utcnow( )) # Bug? duratiion, start_time - end_time TypeError if proc_res.returncode != 0: print_err('%r exited with code %r, stderr:' % (proc.args, proc_res.returncode)) print_err(proc_res.stderr) else: with atomic_save(target_dir + '/.apatite_last_pulled') as f: f.write(cur_dt.isoformat().partition('.')[0].encode('utf8')) return proc_res
def render_report(self): start_state = [{ 'name': k, 'result': v } for k, v in self.start_state.goal_results.items()] start_state.sort(key=lambda g: g['name']) latest_state = [{ 'name': k, 'result': v } for k, v in self.latest_state.goal_results.items()] latest_state.sort(key=lambda g: g['name']) combined = [{ 'start': s[0], 'latest': s[1] } for s in izip_longest(start_state, latest_state)] # TODO: Also combine goals, so you can show info about targets, etc. ctx = { 'id': self.id, 'name': self.name, 'lang': self.lang, 'description': self.description, 'contacts': self.contacts, 'wikiproject_name': self.wikiproject_name, 'campaign_start_date': self.campaign_start_date.isoformat(), 'campaign_end_date': self.campaign_end_date.isoformat(), 'date_created': self.date_created.isoformat(), 'date_updated': datetime.datetime.utcnow().strftime(UPDATED_DT_FORMAT), 'goals': self.goals, 'article_count': len(self.article_title_list), 'start_state_goal': start_state, 'latest_state_goal': latest_state, 'combined_state': combined } campaign_static_path = STATIC_PATH + 'campaigns/%s/' % self.id mkdir_p(campaign_static_path) report_html = ASHES_ENV.render('campaign.html', ctx) report_path = campaign_static_path + 'index.html' report_json_path = campaign_static_path + 'campaign.json' with atomic_save(report_path) as html_f, atomic_save( report_json_path) as json_f: html_f.write(report_html) json.dump(ctx, json_f, indent=2, sort_keys=True) return
def normalize(plist, pfile): """normalize project and tag order, checking for duplicates and format divergences, overwrites the yaml listing""" plist.normalize() new_yaml = plist.to_yaml() # say no to trailing whitespace new_yaml = '\n'.join([line.rstrip() for line in new_yaml.splitlines()]) with atomic_save(pfile) as f: f.write(new_yaml.encode('utf8')) return
def render_article_list(self): all_results = self._get_all_results() for goal in self.goals: goal['slug'] = slugify(goal['name']) ctx = { 'name': self.name, 'lang': self.lang, 'description': self.description, 'contacts': self.contacts, 'wikiproject_name': self.wikiproject_name, 'campaign_start_date': self.campaign_start_date.isoformat(), 'campaign_end_date': self.campaign_end_date.isoformat(), 'date_created': self.date_created.isoformat(), 'date_updated': datetime.datetime.utcnow().strftime(UPDATED_DT_FORMAT), 'article_count': len(self.article_title_list), 'all_results': all_results, 'goals': [{ 'name': 'Article', 'slug': 'title' }] + sorted(self.goals, key=lambda s: s['name']) } campaign_static_path = STATIC_PATH + 'campaigns/%s/' % self.id article_list_html = ASHES_ENV.render('articles.html', ctx) article_list_path = campaign_static_path + 'articles.html' article_list_json_path = campaign_static_path + 'articles.json' mkdir_p(os.path.split(article_list_path)[0]) with atomic_save(article_list_path) as html_f, atomic_save( article_list_json_path) as json_f: html_f.write(article_list_html.encode('utf-8')) json.dump(ctx, json_f, indent=2, sort_keys=True) return
def append_credentials(pgpass, dbcreds): """ Append credentials to pgpass file """ try: with pgpass.open() as fin: data = fin.read() except IOError: data = '' with atomic_save(str(pgpass.absolute()), file_perms=0o600, text_mode=True) as fout: if data: fout.write(data) if not data.endswith('\n'): fout.write('\n') fout.write(':'.join(dbcreds) + '\n')
def _main(): start_time = time.time() with open(PROJ_PATH + '/projects.yaml') as f: projects = yaml.load(f)['projects'] #projects = [p for p in projects if p['name'] == 'scikit-learn'] #if not projects: # return try: with open(PROJ_PATH + '/projects.json') as f: cur_data = json.load(f) cur_projects = cur_data['projects'] cur_gen_date = isoparse(cur_data['gen_date']) except (IOError, KeyError): cur_projects = [] cur_gen_date = None if cur_gen_date: fetch_outdated = (datetime.datetime.utcnow() - cur_gen_date) > datetime.timedelta(seconds=3600) else: fetch_outdated = True cur_names = sorted([c['name'] for c in cur_projects]) new_names = sorted([n['name'] for n in projects]) tpr = os.getenv('TRAVIS_PULL_REQUEST') if tpr and tpr != 'false': print('Pull request detected. Skipping data update until merged.') return if fetch_outdated or cur_names != new_names or os.getenv( 'ZV_DISABLE_CACHING'): entries = fetch_entries(projects) else: print('Current data already up to date, exiting.') return pprint(entries) res = { 'projects': entries, 'gen_date': datetime.datetime.utcnow().isoformat(), 'gen_duration': time.time() - start_time } with atomic_save(PROJ_PATH + '/projects.json') as f: f.write( json.dumps(res, indent=2, sort_keys=True, default=_json_default)) return
def append_credentials(pgpass, dbcreds): """ Append credentials to pgpass file """ try: with pgpass.open() as fin: data = fin.read() except IOError: data = '' # The permissions on .pgpass must disallow any access to world or group. # Hence, chmod 0600 ~/.pgpass. If the permissions are less strict than this, the file will be ignored. with atomic_save(str(pgpass.absolute()), file_perms=0o600, text_mode=True) as fout: if data: fout.write(data) if not data.endswith('\n'): fout.write('\n') fout.write(':'.join(dbcreds) + '\n') print('\nUpdated DEA Database Password in ~/.pgpass file.')
def write_yaml(self, stat: OutputProduct, tmp_filename: Path): output_filename = self.output_filename_tmpname[tmp_filename] datasets = self._find_source_datasets(stat, uri=output_filename.as_uri()) yaml_filename = str(output_filename.with_suffix('.yaml')) # Write to Yaml if len( datasets ) == 1: # I don't think there should ever be more than 1 dataset in here... _LOG.info('writing dataset yaml for %s to %s', stat, yaml_filename) with fileutils.atomic_save(yaml_filename) as yaml_dst: yaml_dst.write(datasets.values[0]) else: _LOG.error( 'Unexpected more than 1 dataset %r being written at once, ' 'investigate!', datasets)
def create_maildir_directories(basedir, is_folder=False): os_makedirs(basedir, 0o700, exist_ok=True) new_path = None for subdir_name in ('tmp', 'cur', 'new'): subdir_path = os.path.join(basedir, subdir_name) os_makedirs(subdir_path, 0o700, exist_ok=True) if subdir_name == 'new': new_path = subdir_path # The maildir++ description [1] mentions a "maildirfolder" file for each # subfolder. Dovecot does not create such a file but doing so seems # harmless. # http://www.courier-mta.org/imap/README.maildirquota.html if is_folder: maildirfolder_path = os.path.join(basedir, 'maildirfolder') # never overwrite an existing "maildirfolder" file (just being overcautious) # in Python 3 we could also use "open(..., 'xb')" and catch FileExistsError with atomic_save(maildirfolder_path, overwrite=False) as fp: pass return new_path
def urlretrieve(self, url, dest, print_progress=True): resp = self._session.get(url, stream=True) total_size, total_chunks = 0, 0 with fileutils.atomic_save(dest) as f: content_iter = resp.iter_content(1024) while 1: size_str = bytes2human(total_size, 1).rjust(7) msg = ('%s downloaded from %s\r' % (size_str, url)) if print_progress and (total_chunks % 20) == 0: sys.stdout.write(msg) sys.stdout.flush() try: chunk = next(content_iter) except StopIteration: if print_progress: sys.stdout.write('\n') break total_size += len(chunk) total_chunks += 1 f.write(chunk) return
def dataset_to_geotif_yaml(dataset: xarray.Dataset, odc_dataset: Dataset, filename: Union[Path, str], variable_params=None): """ Write the dataset out as a set of geotifs with metadata in a yaml file. There will be one geotiff file per band. The band name is added into the file name. i.e ls8_fc.tif -> ls8_fc_BS.tif :param dataset: :param filename: Output filename :param variable_params: dict of variable_name: {param_name: param_value, [...]} Used to get band names. """ bands = variable_params.keys() abs_paths, _, yml = tif_filenames(filename, bands) Path(filename).parent.mkdir(parents=True, exist_ok=True) # Write out the yaml file with fileutils.atomic_save(str(yml)) as stream: yaml.safe_dump(odc_dataset.metadata_doc, stream, encoding='utf8') # Iterate over the bands for key, bandfile in abs_paths.items(): slim_dataset = dataset[[key]] # create a one band dataset attrs = slim_dataset[key].attrs.copy() # To get nodata in del attrs['crs'] # It's format is poor del attrs['units'] # It's format is poor slim_dataset[key] = dataset.data_vars[key].astype('int16', copy=True) write_geotiff(bandfile, slim_dataset.isel(time=0), profile_override=attrs)
def write(self): # TODO: need way to get contents. 'write contents to file' contents = self.get_contents() with atomic_save(self.path) as file: file.write(contents) return
def save(path, data): with atomic_save(path, text_mode=True) as f: json.dump(data, f)
def pypier(args, reqs): ''' only supports pure-python repos for now ''' parser = argparse.ArgumentParser(prog='pypier') parser.add_argument('cmd', choices=('config', 'publish', 'pip-index') + PIP_CMDS) cmd = parser.parse_args(args[1:2]).cmd cache = reqs.cache executor = reqs.executor site_config = reqs.site_config pypier_repo = site_config['pypier']['repo'] pypier_repo_ro = site_config['pypier']['repo_ro'] if cmd == 'config': print print 'PyPIER repos:' print ' ', pypier_repo_ro, '(fetch)' print ' ', pypier_repo, '(publish)' elif cmd == 'publish': parser.add_argument('--dry-run', action='store_true') arg_vals = parser.parse_args(args[1:]) setup_dir = find_project_dir(os.getcwd(), 'setup.py') pypier_read_write = cache.workon_project_git('pypier', pypier_repo) executor.python('setup.py', 'sdist').redirect(cwd=setup_dir) # TODO manylinux wheels? OSX wheels? version = executor.python('setup.py', version=None).batch()[0].strip() output = [ fn for fn in os.listdir(setup_dir + '/dist/') if version in fn ] name = output[0].split('-', 1)[0] # typical artifact: foo-ver.tar.gz dst = pypier_read_write.path + '/packages/' + name + '/' fileutils.mkdir_p(dst) # TODO: instead of just looking for anything in the dist # directory, query setup for the version and check for that. for result in output: if os.path.exists(os.path.join(dst, result)): raise EnvironmentError( "{} has already been published".format(result)) for result in output: shutil.copy(setup_dir + '/dist/' + result, dst) with fileutils.atomic_save(os.path.join(dst, 'pkg_info.json')) as f: pkg_info = get_pkg_info(executor, setup_dir) pkg_info_json = json.dumps(pkg_info, indent=2, sort_keys=True) f.write(pkg_info_json + '\n') update_index(pypier_read_write.path) source_metadata = get_source_metadata(executor, setup_dir) commit_msg = 'PyPIER publish: {}\n\n{}\n'.format( ', '.join(output), json.dumps(source_metadata, indent=2, sort_keys=True)) pypier_read_write.push(commit_msg, dry_run=arg_vals.dry_run) elif cmd == 'pip-index': pypier_read_only = cache.pull_project_git('pypier', pypier_repo_ro) link_path = pypier_read_only + '/packages/index.html' print link_path # NOTE: this print command is the primary purpose elif cmd in PIP_CMDS: pypier_read_only = cache.pull_project_git('pypier', pypier_repo_ro) link_path = pypier_read_only + '/packages/index.html' #env = dict(os.environ) #env['PIP_FIND_LINKS'] = ' '.join( # [link_path] + env.get('PIP_FIND_LINKS', '').split()) # TODO: figure out clean way to extend env # TODO: remove ALL_PROXY='' once urllib3 + requests # do a release and don't pre-emptively die # on socks5h:// proxy executor.patch_env(PIP_FIND_LINKS=link_path, ALL_PROXY='').command(['python', '-m', 'pip'] + args[1:]).redirect( stdout=sys.stdout, stderr=sys.stderr) else: # argparse should catch this above raise ValueError('unrecognized sub-command %r' % cmd)
def write(self): 'write contents to file' contents = self.get_contents() with atomic_save(self.path) as file: file.write(contents.encode('utf8')) return
async def main(loop): download_timer = Timer() extract_timer = Timer() mod_converge_timer = Timer() apply_converge_timer = Timer() purge_timer = Timer() preproc_timer = Timer() postproc_timer = Timer() load_order_timer = Timer() enable_plugins_timer = Timer() mod_list = [ # Install outside of Data/ FastExit(), FourGBPatch(), OBSE(), ENB(), ENBoost(), MoreHeap(), # OBSE Plugins OneTweak(), # OBSETester(), MenuQue(), ConScribe(), Pluggy(), NifSE(), # Performance Streamline(), OSR(), # Necessary Tweaks ATakeAllAlsoBooks(), DarnifiedUI(), DarnifiedUIConfigAddon(), HarvestFlora(), HarvestContainers(), # Textures QTP3R(), GraphicImprovementProject(), ZiraHorseCompilationModpack(), RingRetexture(), KafeisArmoredCirclets(), KoldornsSewerTextures2(), KoldornsCaveTextures2(), MEAT(), BomretTexturePackForShiveringIslesWithUSIP(), # Gameplay RealisticLeveling(), HUDStatusBars(), UV3(), # Install Last INITweaks(), ArchiveInvalidationInvalidated(), ] converged_paths = {} path_mod_owner = {} # which mod owns which path for path in recurse_files(Config.VANILLA_DIR): converged_paths[str(path).lower()] = Config.VANILLA_DIR / path log.info('downloading') with download_timer: for mod in mod_list: async with aiohttp.ClientSession(loop=loop) as session: await mod.download(session) if False: # stop after download? log.info('stopping after download') return log.info('extracting') with extract_timer: for mod in mod_list: await mod.extract() if False: # stop after extract? log.info('stopping after extract') return log.info('pre-processing') with preproc_timer: for mod in mod_list: await mod.preprocess() log.info('calulcating convergance for each mod') with mod_converge_timer: for mod in mod_list: log.info(f'converging {mod.mod_name}') for source_path, dest_path in mod.modify(): if not isinstance(dest_path, Path): raise Exception(f'{dest_path} is not a Path!') elif dest_path.is_absolute(): raise Exception(f'{dest_path} is not absolute') if not isinstance(source_path, Path): raise Exception(f'{source_path} is not a Path!') elif not source_path.is_absolute(): raise Exception(f'{source_path} is not absolute') elif not source_path.exists(): raise Exception(f'{source_path} does not exist, bad modify code') elif not source_path.is_file(): raise Exception(f'{source_path} is not a regular file.') converged_paths[str(dest_path).lower()] = source_path path_mod_owner[str(dest_path).lower()] = mod.mod_name log.info('applying convergance') with apply_converge_timer: for dest_path, source_path in converged_paths.items(): dest_path = Config.game.root_dir / dest_path if not dest_path.exists() or not samefile(str(dest_path), str(source_path)): if dest_path.exists(): dest_path.unlink() # FIXME move to purged dir? dest_path.parent.mkdir(exist_ok=True, parents=True) try: create_hardlink(str(source_path), str(dest_path)) except FileNotFoundError: raise Exception(f'failed to hard link {source_path} to {dest_path} {source_path} (or {dest_path.parent}) not found') log.info('purging') with purge_timer: try: with Config.mod_ownership_path.open('rb') as f: old_path_mod_owner = json.load(f) except FileNotFoundError: old_path_mod_owner = {} purged_root = Config.PURGED_DIR / datetime.now().isoformat().replace(':', '') for path in recurse_files(Config.game.root_dir): if ( str(path).lower() not in converged_paths and not path.suffix.lower() in {'.ini', '.cfg', '.json', '.log'} and #TODO don't purge xml files unless they're menu files not path.parts[0].lower() in {'obmm', 'mopy'} ): if str(path).lower() in old_path_mod_owner: (Config.game.root_dir / path).unlink() else: purged_path = purged_root / path purged_path.parent.mkdir(exist_ok=True, parents=True) (Config.game.root_dir / path).rename(purged_path) log.info('purging empty directories') for d in recurse_dirs(Config.game.root_dir): if dir_is_empty(d): d.rmdir() log.info('postprocessing') with postproc_timer: for mod in mod_list: await mod.postprocess() log.info('Done Applying Changes') log.info('modifying load order') with load_order_timer: boss_uninstall_string = get_regkey('HKLM', r'SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\BOSS', 'UninstallString') boss_install_location = Path(shlex.split(boss_uninstall_string)[0]).parent boss_exe_path = boss_install_location / 'boss.exe' proc = await asyncio.create_subprocess_exec( str(boss_exe_path), '-s', '-g', Config.game.BOSS_NAME, cwd=str(boss_install_location), stderr=sys.stderr, stdout=sys.stdout, ) await proc.wait() log.info('enabling all .esp and .esm files') with enable_plugins_timer: PLUGINS_HEADER = textwrap.dedent(''' # This file is used to tell Oblivion which data files to load. # WRITE YOUR OWN PYTHON SCRIPT TO MODIFY THIS FILE (lol) # Please do not modify this file by hand. ''').strip() with atomic_save(str(Config.game.app_data_path / 'plugins.txt')) as f: with io.TextIOWrapper(f, 'ascii') as ef: ef.write(PLUGINS_HEADER) ef.write('\n') for esm in Config.game.root_dir.glob('Data/*.esm'): ef.write(esm.name) ef.write('\n') for esp in Config.game.root_dir.glob('Data/*.esp'): ef.write(esp.name) ef.write('\n') log.info('saving which mod owns which file') with atomic_save(str(Config.mod_ownership_path)) as f: with io.TextIOWrapper(f, 'ascii') as ef: json.dump(path_mod_owner, ef) log.info(f'download_timer = {download_timer}') log.info(f'extract_timer = {extract_timer}') log.info(f'mod_converge_timer = {mod_converge_timer}') log.info(f'apply_converge_timer = {apply_converge_timer}') log.info(f'purge_timer = {purge_timer}') log.info(f'preproc_timer = {preproc_timer}') log.info(f'postproc_timer = {postproc_timer}') log.info(f'load_order_timer = {load_order_timer}') log.info(f'enable_plugins_timer = {enable_plugins_timer}')
def render(plist, pdir, pfile): "generate the list markdown from the yaml listing" normalize(pfile=pfile, plist=plist) topic_map = plist.get_projects_by_type('topic') topic_toc_text = format_tag_toc(topic_map) projects_by_topic = format_all_categories(topic_map) plat_map = plist.get_projects_by_type('platform') plat_toc_text = format_tag_toc(plat_map) projects_by_plat = format_all_categories(plat_map) context = { 'TOPIC_TOC': topic_toc_text, 'TOPIC_TEXT': projects_by_topic, 'PLATFORM_TOC': plat_toc_text, 'PLATFORM_TEXT': projects_by_plat, 'TOTAL_COUNT': len(plist.project_list) } templates_path = pdir + '/templates/' if not os.path.isdir(templates_path): raise APACLIError('expected "templates" directory at %r' % templates_path) for filename in iter_find_files(templates_path, '*.tmpl.md'): tmpl_text = open(filename).read() target_filename = os.path.split(filename)[1].replace('.tmpl', '') output_text = tmpl_text.format(**context) with atomic_save(pdir + '/' + target_filename) as f: f.write(output_text.encode('utf8')) feed_tmpl_path = templates_path + '/atom.xml' if os.path.exists(feed_tmpl_path): def _stderr_log_func(level, name, message): import sys sys.stderr.write('%s - %s - %s\n' % (level.upper(), name, message)) sys.stderr.flush() ashes_env = AshesEnv([templates_path], log_func=_stderr_log_func) proj_dict_list = [] for proj in plist.project_list: cur = proj.to_dict() cur['name_slug'] = proj.name_slug cur['date_added_utc'] = proj.date_added.isoformat() + 'Z' cur['urls'] = get_url_list(proj) proj_dict_list.append(cur) cur_dt = datetime.datetime.utcnow().replace( microsecond=0).isoformat() + 'Z' res = ashes_env.render( 'atom.xml', { 'projects': sorted(proj_dict_list, key=lambda x: x['date_added'], reverse=True), 'last_generated_utc': cur_dt }) with atomic_save(pdir + '/atom.xml') as f: f.write(res.encode('utf8')) return