def main():
logging.basicConfig(level=logging.INFO)
# Hide messages caused by eve-emdr.com not supporting keep-alive
logging.getLogger("requests").setLevel(logging.WARN)
s = StatsCollector()
sw = StatsWriter(s)
s.start()
sw.start()
t = Trawler(s)
u = EMDRUploader(s)
t.addListener(u)
u.start()
t.trawlMarket()
def main():
logging.basicConfig(level=logging.INFO)
# Hide messages caused by eve-emdr.com not supporting keep-alive
logging.getLogger("requests").setLevel(logging.WARN)
s = StatsCollector()
sw = StatsWriter(s)
s.start()
sw.start()
t = Trawler(s)
if not getenv("DISABLE_EMDR", False):
u = EMDRUploader(s)
t.addListener(u)
u.start()
if "POSTGRES_USERNAME" in os.environ:
p = PostgresAdapter(s)
t.addListener(p)
p.start()
sdw = StatsDBWriter(s)
sdw.start()
t.trawlMarket()
def init_stats():
'''Initialise a StatsCollector for the application to use.'''
stattr = [
# rsync-style 'savings' measurement.
'bytes_total',
'bytes_transferred',
'metadata_bytes_transferred',
# fetch_contents()
'content_fetches',
'metadata_fetches',
# Fetch stats.
'file_contents_differed',
'file_metadata_differed',
'directory_metadata_differed',
'link_contents_differed',
'link_metadata_differed',
]
return StatsCollector.init('AppStats', stattr)
"""
import os
import asgineer
try:
from stats import StatsCollector, UdpStatsReceiver
from stats import stats_handler
except ImportError:
from mypaas.stats import StatsCollector, UdpStatsReceiver
from mypaas.stats import stats_handler
# Create a stats collector
db_dir = os.path.expanduser("~/_stats")
collector = StatsCollector(db_dir)
# Start a thread that receives stats via udp and puts it into the collector
udp_stats_receiver = UdpStatsReceiver(collector)
udp_stats_receiver.start()
@asgineer.to_asgi
async def main_handler(request):
return await stats_handler(request, collector)
if __name__ == "__main__":
asgineer.run(main_handler, "uvicorn", "0.0.0.0:80", log_level="warning")
def fetch_needed(needed, source, opts):
'''
Download/copy the necessary files from opts.source_url or opts.source_dir
to opts.dest_dir.
This is a bit fiddly, as it tries hard to get the modes right.
Returns a tuple (fetch_added, error_count), a list of FileHash objects we
/added/ as a result of this run, and the number of errors in the run.
'''
r = SystemRandom()
fetch_added = []
error_count = 0
if not source.endswith('/'):
source += "/"
counters = StatsCollector('FetchStats', [
'contents_differ_count', 'differing_file_index', 'signature_checkpoint'
])
counters.contents_differ_count = 0
counters.differing_file_index = 0
counters.signature_checkpoint = 0
included_dirs = set()
if opts.include:
log.debug("Includes: %s", opts.include)
re_globmatch = re.compile(r'[*?\[\]]')
incset = set([d for d in opts.include if not re_globmatch.search(d)])
incset_glob = set([d for d in opts.include if d not in incset])
else:
incset = set()
incset_glob = set()
# These are used to debug the -I filter.
i_fetched = []
i_not_fetched = []
# Set up counters for the progress meter.
for fh in needed:
includeable = is_path_included(fh.fpath,
incset,
incset_glob,
included_dirs,
is_dir=fh.is_dir)
if fh.dest_missing or fh.contents_differ:
if not opts.include or includeable:
if fh.is_file:
counters.contents_differ_count += 1
# This is used to get current information into the destination
# hashlist. That way, current information is written to the client
# HSYNC.SIG, saving on re-scans.
changed = StatsCollector('ChangeStatus',
['contents', 'uidgid', 'mode', 'mtime'])
for n, fh in enumerate(needed, start=1):
if opts.include and not is_path_included(
fh.fpath, incset, incset_glob, included_dirs,
is_dir=fh.is_dir):
log.debug("Inclusion filter: '%s' is not included, skipping",
fh.fpath)
i_not_fetched.append(fh)
continue
changed.contents = False
changed.uidgid = False
changed.mode = False
changed.mtime = False
if log.isEnabledFor(logging.DEBUG):
if fh.is_dir:
log.debug("fetch_needed: dir %s", fh.fpath)
else:
log.debug("fetch_needed: %s", fh.fpath)
quoted_fpath = urllib.quote(fh.fpath)
if log.isEnabledFor(logging.DEBUG):
if quoted_fpath != fh.fpath:
log.debug("fetch_needed: escaping '%s' -> '%s'", fh.fpath,
quoted_fpath)
source_url = urlparse.urljoin(source, quoted_fpath)
success = False
if fh.is_file:
try:
_file_fetch(fh, source_url, changed, counters, r, opts)
success = True
except FetchException as e:
log.warn("Fetch file %s failed: %s", source_url, e)
error_count += 1
elif fh.is_link:
try:
_link_fetch(fh, changed, opts)
success = True
except FetchException as e:
log.warn("Update link %s failed: %s", fh.fpath, e)
error_count += 1
elif fh.is_dir:
try:
_dir_fetch(fh, changed, opts)
success = True
except FetchException as e:
log.warn("Update dir %s failed: %s", fh.fpath, e)
error_count += 1
# Update the client-side HSYNC.SIG data.
if success:
i_fetched.append(fh)
log.debug("Updating dest hash for '%s'", fh.fpath)
update_dest_filehash(fh, changed, fetch_added)
else:
i_not_fetched.append(fh)
log.debug("fetch_needed(): done")
if error_count == 0:
if not opts.quiet:
print("Fetch completed")
log.debug("Fetch completed successfully")
else:
log.warn("Fetch completed with %d errors", error_count)
if opts.fetch_debug:
_fetch_debug(i_fetched, i_not_fetched)
return (fetch_added, error_count)
def setUp(self, allowed_names=None):
if not allowed_names:
allowed_names = []
config = get_collector_config('StatsCollector', {})
self.collector = StatsCollector(config, None)
def fetch_needed(needed, source, opts):
'''
Download/copy the necessary files from opts.source_url or opts.source_dir
to opts.dest_dir.
This is a bit fiddly, as it tries hard to get the modes right.
Returns a tuple (fetch_added, error_count), a list of FileHash objects we
/added/ as a result of this run, and the number of errors in the run.
'''
r = SystemRandom()
fetch_added = []
error_count = 0
if not source.endswith('/'):
source += "/"
counters = StatsCollector('FetchStats', [
'contents_differ_count',
'differing_file_index',
'signature_checkpoint'
])
counters.contents_differ_count = 0
counters.differing_file_index = 0
counters.signature_checkpoint = 0
included_dirs = set()
if opts.include:
log.debug("Includes: %s", opts.include)
re_globmatch = re.compile(r'[*?\[\]]')
incset = set([d for d in opts.include if not re_globmatch.search(d)])
incset_glob = set([d for d in opts.include if d not in incset])
else:
incset = set()
incset_glob = set()
# These are used to debug the -I filter.
i_fetched = []
i_not_fetched = []
# Set up counters for the progress meter.
for fh in needed:
includeable = is_path_included(fh.fpath,
incset, incset_glob,
included_dirs,
is_dir=fh.is_dir)
if fh.dest_missing or fh.contents_differ:
if not opts.include or includeable:
if fh.is_file:
counters.contents_differ_count += 1
# This is used to get current information into the destination
# hashlist. That way, current information is written to the client
# HSYNC.SIG, saving on re-scans.
changed = StatsCollector('ChangeStatus',
['contents', 'uidgid', 'mode', 'mtime'])
for n, fh in enumerate(needed, start=1):
if opts.include and not is_path_included(fh.fpath,
incset, incset_glob,
included_dirs,
is_dir=fh.is_dir):
log.debug("Inclusion filter: '%s' is not included, skipping",
fh.fpath)
i_not_fetched.append(fh)
continue
changed.contents = False
changed.uidgid = False
changed.mode = False
changed.mtime = False
if log.isEnabledFor(logging.DEBUG):
if fh.is_dir:
log.debug("fetch_needed: dir %s", fh.fpath)
else:
log.debug("fetch_needed: %s", fh.fpath)
quoted_fpath = urllib.quote(fh.fpath)
if log.isEnabledFor(logging.DEBUG):
if quoted_fpath != fh.fpath:
log.debug("fetch_needed: escaping '%s' -> '%s'",
fh.fpath, quoted_fpath)
source_url = urlparse.urljoin(source, quoted_fpath)
success = False
if fh.is_file:
try:
_file_fetch(fh, source_url, changed, counters, r, opts)
success = True
except FetchException as e:
log.warn("Fetch file %s failed: %s", source_url, e)
error_count += 1
elif fh.is_link:
try:
_link_fetch(fh, changed, opts)
success = True
except FetchException as e:
log.warn("Update link %s failed: %s", fh.fpath, e)
error_count += 1
elif fh.is_dir:
try:
_dir_fetch(fh, changed, opts)
success = True
except FetchException as e:
log.warn("Update dir %s failed: %s", fh.fpath, e)
error_count += 1
# Update the client-side HSYNC.SIG data.
if success:
i_fetched.append(fh)
log.debug("Updating dest hash for '%s'", fh.fpath)
update_dest_filehash(fh, changed, fetch_added)
else:
i_not_fetched.append(fh)
log.debug("fetch_needed(): done")
if error_count == 0:
if not opts.quiet:
print("Fetch completed")
log.debug("Fetch completed successfully")
else:
log.warn("Fetch completed with %d errors", error_count)
if opts.fetch_debug:
_fetch_debug(i_fetched, i_not_fetched)
return (fetch_added, error_count)
