from textwrap import dedent
import logging
import os
import time
import random
import psycopg2 # debdeps: python3-psycopg2
from psycopg2.extras import Json
import ujson
from fastpath.metrics import setup_metrics
log = logging.getLogger("fastpath.db")
metrics = setup_metrics(name="fastpath.db")
conn = None
_autocommit_conn = None
DB_HOST = "hkgmetadb.infra.ooni.io"
DB_USER = "******"
DB_NAME = "metadb"
DB_PASSWORD = "******" # This is already made public
FREE_SPACE_GB = 10.2
def _ping():
q = "SELECT pg_postmaster_start_time();"
with conn.cursor() as cur:
cur.execute(q)
import fastpath.normalize as normalize
from fastpath.metrics import setup_metrics
log = logging.getLogger("fastpath")
# See debian/postinst for ssh keygen
ssh_username = "******"
pkey_filename_local_path = "ssh/id_ed25519"
pkey_password_file = "/etc/machine-id"
collector_hostnames = ("b.collector.ooni.io", "c.collector.ooni.io")
ARCHIVE_DIR = "/srv/collector/archive"
FIND = "/usr/bin/find {} -maxdepth 1 -type f -cmin -{} -printf '%C@ %s %f\n'"
metrics = setup_metrics(name="fastpath.feeder")
# suppress debug logs
for l in ("paramiko", "paramiko.transport"):
logging.getLogger(l).setLevel(logging.WARN)
class Source:
def __init__(self, conf, hostname):
with open(pkey_password_file) as f:
pkey_password = f.read().strip()
pkey_file = conf.vardir / pkey_filename_local_path
assert pkey_file.is_file(), "Missing SSH private key"
log.info("Creating SSH client using %s", pkey_file)
pkey = paramiko.Ed25519Key.from_private_key_file(
pkey_file.as_posix(), password=pkey_password)
import logging
# import lz4.frame as lz4frame # debdeps: python3-lz4
from datetime import datetime, timedelta, timezone
import ujson
import boto3 # debdeps: python3-boto3
try:
from fastpath.metrics import setup_metrics
except ImportError:
from metrics import setup_metrics
log = logging.getLogger("fastpath.s3uploader")
metrics = setup_metrics(name="fastpath.s3uploader")
AWS_PROFILE = "ooni-data"
BUCKET_NAME = "ooni-data"
UPLOAD_PREFIX = "fastpath/"
# suppress debug logs
for l in ("urllib3", "botocore", "s3transfer"):
logging.getLogger(l).setLevel(logging.INFO)
def create_s3_client():
boto3.setup_default_session(profile_name=AWS_PROFILE)
return boto3.client("s3")