def main(): """ Script entry point for pypkgmirror. Parses the configuration and assembles a collection of subprocess calls, then invokes them. """ from pypkgmirror.agents import DebmirrorAgent, RsyncAgent, AptlyAgent if 'loglevel' in conf: log.setLevel(conf['loglevel']) mirrors = [] aptly_mirrors = [] # aptly shares a database so these should not be parallel for _ in conf.get('mirrors', {}).get('debmirror', []): mirrors.append(DebmirrorAgent(_, conf['mirrors']['debmirror'][_])) for _ in conf.get('mirrors', {}).get('rsync', []): mirrors.append(RsyncAgent(_, conf['mirrors']['rsync'][_])) for _ in conf.get('mirrors', {}).get('aptly', []): aptly_mirrors.append(AptlyAgent(_, conf['mirrors']['aptly'][_])) pool = multiprocessing.Pool(2) pool.map(start_sync, mirrors) pool.close() pool.join() pool = multiprocessing.Pool(1) pool.map(start_sync, aptly_mirrors) pool.close() pool.join() _subprocess_call(['hardlink', '-fpot', conf['basedir']])
def start_sync(agent): """ Performs a full mirror update with the given agent. This should typically download any new or updated packages from a remote repository, and update any necessary indexes. """ log.info("Syncing repository '%s' hosted at %s", agent.name, agent.host) outfile_path = "%s/%s.out" % (conf['logdir'], agent.name) errfile_path = "%s/%s.err" % (conf['logdir'], agent.name) mkdir(os.path.dirname(outfile_path)) mkdir(agent.basedir) with open(outfile_path, 'w') as outfile, open(errfile_path, 'w') as errfile: for call in agent.get_calls(): log.debug(' '.join(call)) if conf.get('noop'): continue _subprocess_call(call, outfile, errfile)