def fetch_needed(needed, source, opts): ''' Download/copy the necessary files from opts.source_url or opts.source_dir to opts.dest_dir. This is a bit fiddly, as it tries hard to get the modes right. Returns a tuple (fetch_added, error_count), a list of FileHash objects we /added/ as a result of this run, and the number of errors in the run. ''' r = SystemRandom() fetch_added = [] error_count = 0 if not source.endswith('/'): source += "/" counters = StatsCollector('FetchStats', [ 'contents_differ_count', 'differing_file_index', 'signature_checkpoint' ]) counters.contents_differ_count = 0 counters.differing_file_index = 0 counters.signature_checkpoint = 0 included_dirs = set() if opts.include: log.debug("Includes: %s", opts.include) re_globmatch = re.compile(r'[*?\[\]]') incset = set([d for d in opts.include if not re_globmatch.search(d)]) incset_glob = set([d for d in opts.include if d not in incset]) else: incset = set() incset_glob = set() # These are used to debug the -I filter. i_fetched = [] i_not_fetched = [] # Set up counters for the progress meter. for fh in needed: includeable = is_path_included(fh.fpath, incset, incset_glob, included_dirs, is_dir=fh.is_dir) if fh.dest_missing or fh.contents_differ: if not opts.include or includeable: if fh.is_file: counters.contents_differ_count += 1 # This is used to get current information into the destination # hashlist. That way, current information is written to the client # HSYNC.SIG, saving on re-scans. changed = StatsCollector('ChangeStatus', ['contents', 'uidgid', 'mode', 'mtime']) for n, fh in enumerate(needed, start=1): if opts.include and not is_path_included( fh.fpath, incset, incset_glob, included_dirs, is_dir=fh.is_dir): log.debug("Inclusion filter: '%s' is not included, skipping", fh.fpath) i_not_fetched.append(fh) continue changed.contents = False changed.uidgid = False changed.mode = False changed.mtime = False if log.isEnabledFor(logging.DEBUG): if fh.is_dir: log.debug("fetch_needed: dir %s", fh.fpath) else: log.debug("fetch_needed: %s", fh.fpath) quoted_fpath = urllib.quote(fh.fpath) if log.isEnabledFor(logging.DEBUG): if quoted_fpath != fh.fpath: log.debug("fetch_needed: escaping '%s' -> '%s'", fh.fpath, quoted_fpath) source_url = urlparse.urljoin(source, quoted_fpath) success = False if fh.is_file: try: _file_fetch(fh, source_url, changed, counters, r, opts) success = True except FetchException as e: log.warn("Fetch file %s failed: %s", source_url, e) error_count += 1 elif fh.is_link: try: _link_fetch(fh, changed, opts) success = True except FetchException as e: log.warn("Update link %s failed: %s", fh.fpath, e) error_count += 1 elif fh.is_dir: try: _dir_fetch(fh, changed, opts) success = True except FetchException as e: log.warn("Update dir %s failed: %s", fh.fpath, e) error_count += 1 # Update the client-side HSYNC.SIG data. if success: i_fetched.append(fh) log.debug("Updating dest hash for '%s'", fh.fpath) update_dest_filehash(fh, changed, fetch_added) else: i_not_fetched.append(fh) log.debug("fetch_needed(): done") if error_count == 0: if not opts.quiet: print("Fetch completed") log.debug("Fetch completed successfully") else: log.warn("Fetch completed with %d errors", error_count) if opts.fetch_debug: _fetch_debug(i_fetched, i_not_fetched) return (fetch_added, error_count)
""" import os import asgineer try: from stats import StatsCollector, UdpStatsReceiver from stats import stats_handler except ImportError: from mypaas.stats import StatsCollector, UdpStatsReceiver from mypaas.stats import stats_handler # Create a stats collector db_dir = os.path.expanduser("~/_stats") collector = StatsCollector(db_dir) # Start a thread that receives stats via udp and puts it into the collector udp_stats_receiver = UdpStatsReceiver(collector) udp_stats_receiver.start() @asgineer.to_asgi async def main_handler(request): return await stats_handler(request, collector) if __name__ == "__main__": asgineer.run(main_handler, "uvicorn", "0.0.0.0:80", log_level="warning")
def setUp(self, allowed_names=None): if not allowed_names: allowed_names = [] config = get_collector_config('StatsCollector', {}) self.collector = StatsCollector(config, None)