def entry_to_log_event(entry):
time = entry['_SOURCE_REALTIME_TIMESTAMP'].timestamp()
hw_id = "" if snowflake.snowflake() is None else snowflake.snowflake()
int_map = {'exit_status': entry['EXIT_STATUS']}
normal_map = {'unit': entry['UNIT'],
'exit_code': entry["EXIT_CODE"]}
return LogEntry(category=SERVICE_EXIT_CATEGORY,
time=int(time),
hw_id=hw_id,
normal_map=normal_map,
int_map=int_map)
async def _checkin(self, service_statusmeta):
"""
if previous checkin is successful, create a new channel
(to make sure the channel does't become stale). Otherwise,
keep the existing channel.
"""
if self._checkin_client is None:
chan = ServiceRegistry.get_rpc_channel(
'checkind', ServiceRegistry.CLOUD)
self._checkin_client = CheckindStub(chan)
mconfig = self._service.mconfig
request = CheckinRequest(
gateway_id=snowflake.snowflake(),
system_status=self._system_status(),
platform_info=self._platform_info(),
machine_info=self._machine_info(),
)
logging.debug('Checkin request:\n%s', request)
for statusmeta in service_statusmeta.values():
request.status.meta.update(statusmeta)
try:
await grpc_async_wrapper(
self._checkin_client.Checkin.future(
request, mconfig.checkin_timeout,
),
self._loop)
self._checkin_done()
except grpc.RpcError as err:
self._checkin_error(err)
def sync(self, service_name):
"""
Synchronizes sample queue for specific service to cloud and reschedules
sync loop
"""
if service_name in self._samples_for_service and \
self._samples_for_service[service_name]:
chan = ServiceRegistry.get_rpc_channel(
'metricsd',
ServiceRegistry.CLOUD,
grpc_options=self._grpc_options)
client = MetricsControllerStub(chan)
if self.post_processing_fn:
# If services wants to, let it run a postprocessing function
# If we throw an exception here, we'll have no idea whether
# something was postprocessed or not, so I guess try and make it
# idempotent? #m sevchicken
self.post_processing_fn(
self._samples_for_service[service_name])
samples = self._samples_for_service[service_name]
sample_chunks = self._chunk_samples(samples)
for idx, chunk in enumerate(sample_chunks):
metrics_container = MetricsContainer(
gatewayId=snowflake.snowflake(), family=chunk)
future = client.Collect.future(metrics_container,
self.grpc_timeout)
future.add_done_callback(
self._make_sync_done_func(service_name, idx))
self._samples_for_service[service_name].clear()
self._loop.call_later(self.sync_interval, self.sync, service_name)
def sentry_init(service_name: str,
sentry_mconfig: mconfigs_pb2.SharedSentryConfig) -> None:
"""Initialize connection and start piping errors to sentry.io."""
sentry_status = get_sentry_status(service_name)
if sentry_status == SentryStatus.DISABLED:
return
dsn_python, sample_rate = get_sentry_dsn_and_sample_rate(sentry_mconfig)
if not dsn_python:
logging.info(
'Sentry disabled because of missing dsn_python. '
'See documentation (Configure > AGW) on how to configure '
'Sentry dsn.', )
return
sentry_sdk.init(
dsn=dsn_python,
release=os.getenv(COMMIT_HASH),
traces_sample_rate=sample_rate,
before_send=_ignore_if_not_marked
if sentry_status == SentryStatus.SEND_SELECTED_ERRORS else None,
)
cloud_address = get_service_config_value(
CONTROL_PROXY,
CLOUD_ADDRESS,
default=None,
)
sentry_sdk.set_tag(ORC8R_CLOUD_ADDRESS, cloud_address)
sentry_sdk.set_tag(HWID, snowflake.snowflake())
sentry_sdk.set_tag(SERVICE_NAME, service_name)
def register_update(eapi):
"""Ensures the inbound URL for the BTS is up to date."""
vpn_ip = system_utilities.get_vpn_ip()
vpn_status = "up" if vpn_ip else "down"
# This could fail when offline! Must handle connection exceptions.
try:
params = {
'bts_uuid': snowflake.snowflake(),
'vpn_status': vpn_status,
'vpn_ip': vpn_ip,
# federer always runs on port 80, but didn't in old versions
'federer_port': "80",
}
r = requests.get(conf['registry'] + "/bts/register",
params=params,
headers=eapi.auth_header,
timeout=11)
if r.status_code == 200:
try:
d = json.loads(r.text)
if 'bts_secret' in d:
conf['bts_secret'] = d['bts_secret']
except ValueError:
pass
return r.text
else:
raise ValueError("BTS registration update failed with status"
" %d (%s)" % (r.status_code, r.text))
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
logger.error("register_update failed due to connection error or"
" timeout.")
def main():
parser = argparse.ArgumentParser(
description='Show the UUID and base64 encoded DER public key', )
parser.add_argument(
"--pub_key",
type=str,
default="/var/opt/magma/certs/gw_challenge.key",
)
opts = parser.parse_args()
public_key = load_public_key_to_base64der(opts.pub_key)
msg = textwrap.dedent("""
Hardware ID
-----------
{}
Challenge key
-------------
{}
Notes
-----
- Hardware ID is this gateway's unique identifier
- Challenge key is this gateway's long-term keypair used for
bootstrapping a secure connection to the cloud
""")
print(msg.format(snowflake.snowflake(), public_key.decode('utf-8')))
def GetGatewayId(self, _, context):
"""
Get gateway hardware ID
"""
return magmad_pb2.GetGatewayIdResponse(
gateway_id=snowflake.snowflake(),
)
def _get_checkin_request_as_state(self) -> State:
request = self._checkin_manager.get_latest_checkin_request()
value = MessageToJson(request)
state = State(type="checkin_request",
deviceID=snowflake.snowflake(),
value=value.encode('utf-8'))
return state
def sync(self):
"""
Synchronizes sample queue to cloud and reschedules sync loop
"""
if self._samples:
chan = ServiceRegistry.get_rpc_channel('metricsd',
ServiceRegistry.CLOUD)
client = MetricsControllerStub(chan)
if self.post_processing_fn:
# If services wants to, let it run a postprocessing function
# If we throw an exception here, we'll have no idea whether
# something was postprocessed or not, so I guess try and make it
# idempotent? #m sevchicken
self.post_processing_fn(self._samples)
samples = self._retry_queue + self._samples
metrics_container = MetricsContainer(
gatewayId=snowflake.snowflake(), family=samples)
future = client.Collect.future(metrics_container,
self.grpc_timeout)
future.add_done_callback(
lambda future: self._loop.call_soon_threadsafe(
self.sync_done, samples, future))
self._retry_queue.clear()
self._samples.clear()
self._loop.call_later(self.sync_interval, self.sync)
def sentry_init(service_name: str, sentry_mconfig: mconfigs_pb2.SharedSentryConfig) -> None:
"""Initialize connection and start piping errors to sentry.io."""
sentry_config = _get_shared_sentry_config(sentry_mconfig)
if not sentry_config.dsn:
logging.info(
'Sentry disabled because of missing dsn_python. '
'See documentation (Configure > AGW) on how to configure '
'Sentry dsn.',
)
return
sentry_sdk.init(
dsn=sentry_config.dsn,
release=os.getenv(COMMIT_HASH),
traces_sample_rate=sentry_config.sample_rate,
before_send=_get_before_send_hook(sentry_config.exclusion_patterns),
)
cloud_address = get_service_config_value(
CONTROL_PROXY,
CLOUD_ADDRESS,
default=None,
)
sentry_sdk.set_tag(ORC8R_CLOUD_ADDRESS, cloud_address)
sentry_sdk.set_tag(HWID, snowflake.snowflake())
sentry_sdk.set_tag(SERVICE_NAME, service_name)
def register_subscriber(self, imsi):
"""Send a request to the registry server with this BTS unique ID and
the number.
Raises: ValueError if the API failed to register the user
400 - Bad parameters
403 - User is not associated with this BTS
404 - No numbers available
409 - IMSI already registered to another network
500 - Uh-oh
"""
url = self.conf['registry'] + "/register/"
try:
r = requests.post(url,
headers=self.auth_header,
data={
'imsi': imsi,
'bts_uuid': snowflake.snowflake()
})
except BaseException as e: # log and rethrow
logger.error("Endaga: Register network error: %s." % e)
raise
if r.status_code != 200:
raise ValueError(r.text)
return json.loads(r.text)
def register_handler(client: RegistrationStub, args: List[str]) -> RegisterResponse:
"""
Register a device and retrieves its control proxy
Args:
client: Registration stub
args: command line arguments
Returns:
RegisterRequest: register request, used for printing after function returns
RegisterResponse: response from gRPC call, either error or the control_proxy
"""
req = RegisterRequest(
token=args.token,
hwid=AccessGatewayID(
id=snowflake.snowflake(),
),
challenge_key=ChallengeKey(
key=load_public_key_to_base64der("/var/opt/magma/certs/gw_challenge.key"),
key_type=ChallengeKey.KeyType.SOFTWARE_ECDSA_SHA256,
),
)
res = client.Register(req)
if res.HasField("error"):
raise Exception(res.error)
return req, res
def _package_and_send_metrics(
self, metrics: [metrics_pb2.MetricFamily],
target: ScrapeTarget,
) -> None:
"""
Send parsed and protobuf-converted metrics to cloud.
"""
chan = ServiceRegistry.get_rpc_channel(
'metricsd',
ServiceRegistry.CLOUD,
grpc_options=self._grpc_options,
)
client = MetricsControllerStub(chan)
for chunk in self._chunk_samples(metrics):
metrics_container = MetricsContainer(
gatewayId=snowflake.snowflake(),
family=chunk,
)
future = client.Collect.future(
metrics_container,
self.grpc_timeout,
)
future.add_done_callback(
lambda future:
self._loop.call_soon_threadsafe(
self.scrape_done, future, target,
),
)
self._loop.call_later(
target.interval,
self.scrape_prometheus_target, target,
)
def _get_gw_state(self) -> Optional[State]:
gw_state = self._checkin_manager.get_latest_gw_state()
if gw_state is not None:
state = State(type="gw_state",
deviceID=snowflake.snowflake(),
value=gw_state.encode('utf-8'))
return state
return None
def updated_stored_mconfig():
log_event(
Event(
stream_name="magmad",
event_type="updated_stored_mconfig",
tag=snowflake.snowflake(),
value="{}",
))
def disconnected_sync_rpc_stream():
log_event(
Event(
stream_name="magmad",
event_type="disconnected_sync_rpc_stream",
tag=snowflake.snowflake(),
value="{}"
)
)
def POST(self):
data = web.input()
# The auth token should be here.
logger.info('GET mock subscriber provision')
auth_token = web.ctx.env['HTTP_AUTHORIZATION']
assert data.bts_uuid == snowflake.snowflake()
assert len(data.imsi) == 19
assert auth_token == "Token %s" % CONF['endaga_token']
return json.dumps({ 'number': '1%010d' % random.randint(1000000000, 9999999999) })
def established_sync_rpc_stream():
log_event(
Event(
stream_name="magmad",
event_type="established_sync_rpc_stream",
tag=snowflake.snowflake(),
value="{}"
)
)
def _get_snowflake():
""" Read UUID from /etc/snowflake. If it doesn't exist, die. """
bts_uuid = snowflake()
if bts_uuid:
return bts_uuid
SNOWFLAKE_MISSING = '/etc/snowflake missing'
logger.critical(SNOWFLAKE_MISSING)
raise SystemExit(SNOWFLAKE_MISSING)
def restarted_services(services):
# Convert to a list for JSON serializability
services = list(services)
log_event(
Event(
stream_name="magmad",
event_type="restarted_services",
tag=snowflake.snowflake(),
value=json.dumps(RestartedServices(services=services).to_dict()),
))
def makeSnow(flakes, size, stroke, t):
snowCount = 0
noFlake = 0
t.pensize(stroke)
while (snowCount < flakes):
nextCoord = utils.randomCoord()
if (allowFlake(nextCoord, size)):
noFlake = 0
snowflakes.append(nextCoord)
utils.moveCursor(nextCoord[0], nextCoord[1], t)
t.pencolor(utils.colorPicker())
snowflake.snowflake(size, t)
snowCount += 1
if (snowCount % 3 == 0):
eraseFlake(snowflakes[0], size, t)
else:
noFlake += 1
if (noFlake > 5):
eraseFlake(snowflakes[0], size, t)
def getLocalHostDetails():
while (1):
try:
hostid = snowflake.snowflake()
ipAddr = socket.gethostbyname(socket.gethostname()).strip()
totalCpus = multiprocessing.cpu_count() * 3
return (hostid, ipAddr, totalCpus)
except:
print(str(sys.exc_info()))
time.sleep(1)
def make_scoped_device_id(id, scope):
"""
Create a deviceID of the format <id> for scope 'network'
Otherwise create a key of the format <hwid>:<id> for 'gateway' or
unrecognized scope.
"""
if scope == "network":
return id
else:
return snowflake.snowflake() + ":" + id
def getLocalHostDetails():
while(1):
try:
hostid = snowflake.snowflake()
ipAddr = socket.gethostbyname(socket.gethostname()).strip()
totalCpus = multiprocessing.cpu_count() * 3
return(hostid,ipAddr,totalCpus)
except:
print(str(sys.exc_info()))
time.sleep(1)
def processed_updates(updates):
# Convert updates to dicts for JSON serializability
dict_updates = [MessageToDict(u) for u in updates]
log_event(
Event(
stream_name="magmad",
event_type="processed_updates",
tag=snowflake.snowflake(),
value=json.dumps(ProcessedUpdates(updates=dict_updates).to_dict()),
))
def _randomize_sync_interval(interval: int) -> int:
"""_randomize_sync_interval increases sync interval by random amount.
Increased sync interval ameliorates the thundering herd effect at Orc8r.
"Random" increase is deterministic based on the gateway's HWID.
"""
h = hashlib.md5()
h.update(bytes(snowflake.snowflake(), 'utf8')) # digest of hwid
multiplier = (hash(h.hexdigest()) % 100) / 100 # to interval [0, 1]
delta = multiplier * (interval / 5) # up to 1/5 of target interval
return int(interval + delta)
def main():
parser = argparse.ArgumentParser(
description='Show the UUID and base64 encoded DER public key')
parser.add_argument("--pub_key", type=str,
default="/var/opt/magma/certs/gw_challenge.key")
opts = parser.parse_args()
public_key = load_public_key_to_base64der(opts.pub_key)
print("Hardware ID:\n------------\n%s\n" % snowflake.snowflake())
print("Challenge Key:\n-----------\n%s" % public_key.decode('utf-8'))
def _get_gw_state(self) -> Optional[State]:
gw_type = "gw_state"
gw_state, has_all_required_fields = \
self._gw_status_factory.get_serialized_status()
if has_all_required_fields:
self._error_handler.num_skipped_gateway_states = 0
return self._make_state(gw_type, snowflake.snowflake(), gw_state)
# check if we have failed to send states too many times in a row
if 0 < self._error_handler.max_skipped_gw_states < \
self._error_handler.num_skipped_gateway_states:
logging.warning(
"Number of skipped checkins exceeds %d "
"(cfg: max_skipped_checkins). Checking in anyway.",
self._error_handler.max_skipped_gw_states)
# intentionally don't reset num_skipped_gateway_states here
return self._make_state(gw_type, snowflake.snowflake(), gw_state)
# skipping reporting gateway state
self._error_handler.num_skipped_gateway_states += 1
return None
def processed_updates(configs_by_service):
# Convert to dicts for JSON serializability
configs = {}
for srv, config in configs_by_service.items():
configs[srv] = MessageToDict(config)
log_event(
Event(
stream_name="magmad",
event_type="processed_updates",
tag=snowflake.snowflake(),
value=json.dumps(configs),
))
def process_stream_updates(self, client, stream_name, callback):
extra_args = self._get_extra_args_any(callback, stream_name)
request = StreamRequest(gatewayId=snowflake.snowflake(),
stream_name=stream_name,
extra_args=extra_args)
for update_batch in client.GetUpdates(request,
timeout=self._stream_timeout):
self._loop.call_soon_threadsafe(
callback.process_update,
stream_name,
update_batch.updates,
update_batch.resync,
)
def generate_redirect_log_entry(self, redirect_info):
time = int(datetime.datetime.now().timestamp())
hw_id = snowflake.snowflake()
int_map = {'server_response': redirect_info.server_response.http_code}
normal_map = {
'subscriber_ip': redirect_info.subscriber_ip,
'redirect_address': redirect_info.server_response.redirect_address
}
return LogEntry(category=self.LOGGING_CATEGORY,
time=int(time),
hw_id=hw_id,
normal_map=normal_map,
int_map=int_map)
def check_config(c):
if 'uuid' not in c:
c['uuid'] = snowflake.snowflake()
log.info("My UUID is {}".format(c['uuid']))
write_config(c)
if 'key' not in c:
log.info("Requesting API key from service at {}".format(SERVICE_URL))
response = requests.get( SERVICE_URL + 'key', params={'uuid':c['uuid']})
if response.status_code == 200:
log.info("Got API key, storing it in {}".format(CONFIG_FILE))
response = json.loads(response.text)
c['key'] = response['key']
write_config(c)
else:
log.critical((
"Could not request API key (status {}) "+
"and it is not in the config file."
).format(response.status_code))
error_state("Failed to get an API key. Cannot continue, but will keep flashing fail LED.")
log.info("My configuration is: {}".format(json.dumps(c)))
return c
def checkin(server=conf["server"], data={}):
private_ips = str(get_private_ips())
data["private_ip"] = private_ips
url = server + "/checkin/%s" % snowflake.snowflake()
print url
return requests.get(url, params=data)
import snowflake
import subprocess
import time
import sys
dirSelf = os.path.dirname(os.path.realpath(__file__))
libDir = dirSelf.rstrip(os.sep).rstrip("theBox").rstrip(os.sep) + os.sep + "lib"
sys.path.append(libDir)
import constants
hostname = "google.com" #example
headers = {}
headers['user-agent'] = "theBox-v1.0"
boxid = snowflake.snowflake()
serverHost = "http://"+ constants.backendServer +"/ALIVE"
timeToWait = 30
timeInformed = 0
def getPublicIP():
p = subprocess.Popen(['dig','+short','myip.opendns.com','@resolver1.opendns.com'],stdout=subprocess.PIPE)
t = p.communicate()
p.wait()
if(t):
return(t[0])
else:
return(0)
