def main(shard_num):
maybe_enable_rollbar()
users = config.get_required("DATABASE_USERS")
creds = dict(hostname=None, username=None, password=None, db_name=None)
for database in config.get_required("DATABASE_HOSTS"):
for shard in database["SHARDS"]:
if shard["ID"] == shard_num:
creds["hostname"] = database["HOSTNAME"]
hostname = creds["hostname"]
creds["username"] = users[hostname]["USER"]
creds["password"] = users[hostname]["PASSWORD"]
creds["db_name"] = shard["SCHEMA_NAME"]
break
for key in creds.keys():
if creds[key] is None:
print("Error: {key} is None".format(key=key))
sys.exit(-1)
proc = subprocess.Popen([
"mysql",
"-h" + creds["hostname"],
"-u" + creds["username"],
"-D " + creds["db_name"],
"-p" + creds["password"],
"--safe-updates",
])
proc.wait()
def main(host, port, account_id, folder_id, device_id):
maybe_enable_rollbar()
print("Clearing heartbeat status...")
n = clear_heartbeat_status(account_id, folder_id, device_id, host, port)
print("{} folders cleared.".format(n))
exit(0)
def main(revision_id):
maybe_enable_rollbar()
alembic_ini_filename = os.environ.get("ALEMBIC_INI_PATH", "alembic.ini")
assert os.path.isfile(
alembic_ini_filename), "Missing alembic.ini file at {}".format(
alembic_ini_filename)
database_hosts = config.get_required("DATABASE_HOSTS")
for host in database_hosts:
for shard in host["SHARDS"]:
key = shard["ID"]
if shard.get("DISABLED"):
# Do not include disabled shards since application services
# do not use them.
continue
key = shard["ID"]
try:
print("Stamping shard_id {}".format(key))
alembic_cfg = alembic.config.Config(alembic_ini_filename)
alembic_cfg.set_main_option("shard_id", str(key))
alembic.command.stamp(alembic_cfg, revision_id)
print("Stamped shard_id {}\n".format(key))
except alembic.util.CommandError as e:
print("FAILED to stamp shard_id {} with error: {}".format(
key, str(e)))
continue
def main():
from inbox.config import config, secrets_path
maybe_enable_rollbar()
# If the config contains encryption keys, don't override.
if config.get("SECRET_ENCRYPTION_KEY"):
raise Exception("Encryption keys already present in secrets config "
"file {0}".format(secrets_path))
# Generate keys
data = {
"SECRET_ENCRYPTION_KEY":
binascii.hexlify(nacl.utils.random(nacl.secret.SecretBox.KEY_SIZE)),
"BLOCK_ENCRYPTION_KEY":
binascii.hexlify(nacl.utils.random(nacl.secret.SecretBox.KEY_SIZE)),
}
# Our secrets config file contains our database credentials etc.,
# so it better exist.
# Update it
try:
with open(secrets_path, "a") as f:
print(
"Writing keys to secrets config file {0}".format(secrets_path))
yaml.dump(data, f, default_flow_style=False)
except IOError:
raise Exception(
"Check file write permissions on config file {0}".format(
secrets_path))
# Update the config dict
config.update(data)
def reset_db(dry_run):
maybe_enable_rollbar()
database_hosts = config.get_required("DATABASE_HOSTS")
database_users = config.get_required("DATABASE_USERS")
# Do not include disabled shards since application services do not use them.
engine_manager = EngineManager(database_hosts,
database_users,
include_disabled=False)
for host in database_hosts:
for shard in host["SHARDS"]:
if shard.get("DISABLED"):
continue
key = int(shard["ID"])
engine = engine_manager.engines[key]
schema = shard["SCHEMA_NAME"]
print("Resetting invalid autoincrements for database: {}".format(
schema))
reset_tables = reset_invalid_autoincrements(
engine, schema, key, dry_run)
if dry_run:
print("dry_run=True")
if reset_tables:
print("Reset tables: {}".format(", ".join(reset_tables)))
else:
print("Schema {} okay".format(schema))
def main(host, port, database):
maybe_enable_rollbar()
connection_pool = BlockingConnectionPool(
host=host,
port=port,
db=database,
max_connections=MAX_CONNECTIONS,
timeout=WAIT_TIMEOUT,
socket_timeout=SOCKET_TIMEOUT,
)
client = StrictRedis(host, port, database, connection_pool=connection_pool)
batch_client = client.pipeline()
count = 0
for name in client.scan_iter(count=100):
if name == "ElastiCacheMasterReplicationTimestamp":
continue
batch_client.delete(name)
count += 1
batch_client.execute()
print("{} heartbeats deleted!".format(count))
exit(0)
def main(shard_id, dry_run):
maybe_enable_rollbar()
if shard_id is not None:
backfix_shard(shard_id, dry_run)
else:
for shard_id in engine_manager.engines:
backfix_shard(shard_id, dry_run)
def delete_namespace_indexes(namespace_ids):
"""
Delete the CloudSearch indexes for a list of namespaces, specified by id.
"""
maybe_enable_rollbar()
delete_indexes(namespace_ids)
def console(email_address, client):
""" REPL for Nylas. """
maybe_enable_rollbar()
if client:
start_client_console(email_address)
else:
start_console(email_address)
def main(shard_id, id_start, dry_run):
maybe_enable_rollbar()
if shard_id is not None:
process_shard(shard_id, dry_run, id_start)
else:
for shard_id in engine_manager.engines:
process_shard(shard_id, dry_run, id_start)
def main(shard_id):
maybe_enable_rollbar()
if shard_id is not None:
update_categories_for_shard(shard_id)
update_folders_and_labels_for_shard(shard_id)
else:
for key in engine_manager.engines:
update_categories_for_shard(key)
update_folders_and_labels_for_shard(key)
def main(hostname):
maybe_enable_rollbar()
with global_session_scope() as db_session:
account_ids = db_session.query(Account.id).filter(Account.sync_host == hostname)
print("Accounts being synced by {}:".format(hostname))
for account_id in account_ids:
print(account_id[0])
db_session.commit()
def main():
maybe_enable_rollbar()
setproctitle("scheduler")
zones = {h.get("ZONE") for h in config["DATABASE_HOSTS"]}
threads = []
for zone in zones:
populator = QueuePopulator(zone)
threads.append(gevent.spawn(populator.run))
gevent.joinall(threads)
def main(namespace_ids):
"""
Idempotently index the given namespace_ids.
"""
maybe_enable_rollbar()
for namespace_id in namespace_ids:
log.info("indexing namespace {namespace_id}".format(
namespace_id=namespace_id))
index_namespace(namespace_id)
def main(host, port):
maybe_enable_rollbar()
if host:
status_client = _get_redis_client(host, port, STATUS_DATABASE)
report_client = _get_redis_client(host, port, REPORT_DATABASE)
else:
status_client = get_redis_client(STATUS_DATABASE)
report_client = get_redis_client(REPORT_DATABASE)
status_client.flushdb()
report_client.flushdb()
exit(0)
def run(throttle, dry_run):
maybe_enable_rollbar()
print("Python", sys.version, file=sys.stderr)
pool = []
for host in config["DATABASE_HOSTS"]:
log.info("Spawning delete process for host", host=host["HOSTNAME"])
pool.append(gevent.spawn(delete_account_data, host, throttle, dry_run))
gevent.joinall(pool)
def main(min_id, max_id, shard_id):
maybe_enable_rollbar()
generic_accounts = []
failed = []
if min_id is not None or max_id is not None:
# Get the list of running Gmail accounts.
with global_session_scope() as db_session:
generic_accounts = db_session.query(GenericAccount).filter(
GenericAccount.sync_state == "running")
if min_id is not None:
generic_accounts = generic_accounts.filter(
GenericAccount.id > min_id)
if max_id is not None:
generic_accounts = generic_accounts.filter(
GenericAccount.id <= max_id)
generic_accounts = [acc.id for acc in generic_accounts]
db_session.expunge_all()
elif shard_id is not None:
with session_scope_by_shard_id(shard_id) as db_session:
generic_accounts = db_session.query(GenericAccount).filter(
GenericAccount.sync_state == "running")
generic_accounts = [acc.id for acc in generic_accounts]
db_session.expunge_all()
print("Total accounts", len(generic_accounts))
for account_id in generic_accounts:
try:
with session_scope(account_id) as db_session:
account = db_session.query(GenericAccount).get(account_id)
print("Updating", account.email_address)
with connection_pool(account.id).get() as crispin_client:
account.folder_prefix = crispin_client.folder_prefix
account.folder_separator = crispin_client.folder_separator
db_session.commit()
except Exception:
failed.append(account_id)
print("Processed accounts:")
print(generic_accounts)
print("Failed accounts:")
print(failed)
def delete_account_data(account_id, dry_run, yes, throttle):
maybe_enable_rollbar()
with session_scope(account_id) as db_session:
account = db_session.query(Account).get(account_id)
if not account:
print("Account with id {} does NOT exist.".format(account_id))
return
email_address = account.email_address
namespace_id = account.namespace.id
if account.sync_should_run or not account.is_marked_for_deletion:
print("Account with id {} NOT marked for deletion.\n"
"Will NOT delete, goodbye.".format(account_id))
return -1
if not yes:
question = (
"Are you sure you want to delete all data for account with "
"id: {}, email_address: {} and namespace_id: {}? [yes / no]".
format(account_id, email_address, namespace_id))
answer = raw_input(question).strip().lower()
if answer != "yes":
print("Will NOT delete, goodbye.")
return 0
print("Deleting account with id: {}...".format(account_id))
start = time.time()
# Delete data in database
try:
print("Deleting database data")
delete_namespace(namespace_id, dry_run=dry_run, throttle=throttle)
except Exception as e:
print("Database data deletion failed! Error: {}".format(str(e)))
return -1
database_end = time.time()
print("Database data deleted. Time taken: {}".format(database_end - start))
# Delete liveness data
print("Deleting liveness data")
clear_heartbeat_status(account_id)
end = time.time()
print("All data deleted successfully! TOTAL time taken: {}".format(end -
start))
return 0
def run(days_ago, limit, throttle, dry_run):
maybe_enable_rollbar()
print("Python", sys.version, file=sys.stderr)
pool = []
for host in config["DATABASE_HOSTS"]:
pool.append(
gevent.spawn(purge_old_transactions, host, days_ago, limit,
throttle, dry_run))
gevent.joinall(pool)
def main(dry_run, number, hostname, process):
"""
Unschedule all accounts assigned to a given sync host.
Intended primarily for use when decomissioning sync instances or for
manually unloading an overloaded sync instance.
"""
maybe_enable_rollbar()
if not number:
message = (
"You have not provided a --number option. This will "
"unschedule ALL syncs on the host. Proceed? [Y/n] "
)
if raw_input(message).strip().lower() == "n":
print("Will not proceed")
return
if not dry_run:
message = (
"It is unsafe to unassign hosts while mailsync processes are running. "
"Have you shut down the appropriate mailsync processes on {}? [Y/n]".format(
hostname
)
)
if raw_input(message).strip().lower() == "n":
print("Bailing out")
return
with global_session_scope() as db_session:
if process is not None:
hostname = ":".join([hostname, process])
to_unschedule = db_session.query(Account.id).filter(
Account.sync_host.like("{}%".format(hostname))
)
if number:
to_unschedule = to_unschedule.limit(number)
to_unschedule = [id_ for id_, in to_unschedule.all()]
if number:
to_unschedule = to_unschedule[:number]
for account_id in to_unschedule:
with session_scope(account_id) as db_session:
if dry_run:
print("Would unassign", account_id)
else:
account = db_session.query(Account).get(account_id)
print("Unassigning", account.id)
account.desired_sync_host = None
account.sync_host = None
db_session.commit()
def main(prod, start_syncback, enable_tracer, config, port, enable_profiler):
""" Launch the Nylas API service. """
level = os.environ.get("LOGLEVEL", inbox_config.get("LOGLEVEL"))
configure_logging(log_level=level)
maybe_enable_rollbar()
if config is not None:
config_path = os.path.abspath(config)
load_overrides(config_path)
if prod:
start(port, start_syncback, enable_tracer, enable_profiler)
else:
preflight()
from werkzeug.serving import run_with_reloader
run_with_reloader(lambda: start(port, start_syncback, enable_tracer,
enable_profiler))
def main():
maybe_enable_rollbar()
database_hosts = config.get_required("DATABASE_HOSTS")
database_users = config.get_required("DATABASE_USERS")
# Do not include disabled shards since application services do not use them.
engine_manager = EngineManager(database_hosts,
database_users,
include_disabled=False)
for host in database_hosts:
for shard in host["SHARDS"]:
if shard.get("DISABLED"):
continue
key = int(shard["ID"])
engine = engine_manager.engines[key]
schema = shard["SCHEMA_NAME"]
print("Verifying database: {}".format(schema))
verify_db(engine, schema, key)
def main(account_id, desired_host, dry_run, toggle_sync):
maybe_enable_rollbar()
with global_session_scope() as db_session:
account = db_session.query(Account).get(int(account_id))
print("Before sync host: {}".format(account.sync_host))
print("Before desired sync host: {}".format(account.desired_sync_host))
print("Before sync should run: {}".format(account.sync_should_run))
if dry_run:
return
account.desired_sync_host = desired_host
if toggle_sync:
account.sync_should_run = not account.sync_should_run
print("After sync host: {}".format(account.sync_host))
print("After desired sync host: {}".format(account.desired_sync_host))
print("After sync should run: {}".format(account.sync_should_run))
db_session.commit()
def main():
"""
Detects accounts with sync_state and sync_host inconsistent with
sync_should_run bit. (At one point, this could happen if, say, an account
was _started_ on a new host without being first stopped on its previous
host.)
"""
maybe_enable_rollbar()
with global_session_scope() as db_session:
for acc in (db_session.query(Account).options(
load_only("sync_state", "sync_should_run", "sync_host",
"desired_sync_host")).filter(
Account.sync_state == "stopped")):
if acc.desired_sync_host is not None:
print("account {} assigned to {} but has sync_state 'stopped'"
" ({}, {})".format(acc.id, acc.sync_host,
acc.sync_should_run, acc.sync_host))
def main(prod, config, process_num, syncback_id, enable_tracer,
enable_profiler):
""" Launch the actions syncback service. """
setproctitle("syncback-{}".format(process_num))
maybe_enable_rollbar()
print("Python", sys.version, file=sys.stderr)
if config is not None:
config_path = os.path.abspath(config)
load_overrides(config_path)
level = os.environ.get("LOGLEVEL", inbox_config.get("LOGLEVEL"))
configure_logging(log_level=level)
reconfigure_logging()
total_processes = int(os.environ.get("SYNCBACK_PROCESSES", 1))
def start():
# Start the syncback service, and just hang out forever
syncback = SyncbackService(syncback_id, process_num, total_processes)
if enable_profiler:
inbox_config["DEBUG_PROFILING_ON"] = True
port = 16384 + process_num
enable_profiler_api = inbox_config.get("DEBUG_PROFILING_ON")
frontend = SyncbackHTTPFrontend(port, enable_tracer,
enable_profiler_api)
frontend.start()
syncback.start()
syncback.join()
if prod:
start()
else:
preflight()
from werkzeug.serving import run_with_reloader
run_with_reloader(start)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"-u", "--with-users", action="store_true", dest="with_users", default=False
)
args = parser.parse_args()
from inbox.ignition import init_db, main_engine
maybe_enable_rollbar()
engine = main_engine(pool_size=1)
# Always keep the 'alembic_version' table
keep_tables = ["alembic_version"]
reset_columns = {}
# '--with-users' NOT specified
if not args.with_users:
keep_tables += [
"user",
"namespace",
"account",
"imapaccount",
"user_session",
"easaccount",
"folder",
"gmailaccount",
"outlookaccount",
"genericaccount",
"secret",
"calendar",
"easdevice",
]
reset_columns = {"easaccount": ["eas_account_sync_key", "eas_state"]}
drop_everything(engine, keep_tables=keep_tables, reset_columns=reset_columns)
# recreate dropped tables
init_db(engine)
sys.exit(0)
def main(email_address, reauth, target, provider):
""" Auth an email account. """
preflight()
maybe_enable_rollbar()
shard_id = target << 48
with session_scope(shard_id) as db_session:
account = (db_session.query(Account).filter_by(
email_address=email_address).first())
if account is not None and not reauth:
sys.exit("Already have this account!")
if not provider:
provider = provider_from_address(email_address)
# Resolve unknown providers into either custom IMAP or EAS.
if provider == "unknown":
is_imap = raw_input(
"IMAP account? [Y/n] ").strip().lower() != "n"
provider = "custom" if is_imap else "eas"
auth_handler = handler_from_provider(provider)
account_data = auth_handler.interactive_auth(email_address)
if reauth:
account = auth_handler.update_account(account, account_data)
else:
account = auth_handler.create_account(account_data)
try:
if auth_handler.verify_account(account):
db_session.add(account)
db_session.commit()
except NotSupportedError as e:
sys.exit(str(e))
print("OK. Authenticated account for {}".format(email_address))
def main(num_accounts):
maybe_enable_rollbar()
with global_session_scope() as db_session:
accounts = (db_session.query(Account).filter(
Account.sync_should_run == true()).order_by(
func.rand()).limit(num_accounts).all())
accounts = [acc.id for acc in accounts][:num_accounts]
db_session.expunge_all()
pool = Pool(size=100)
results = pool.map(process_account, accounts)
global_results = dict()
for ret in results:
for key in ret:
if key not in global_results:
global_results[key] = 0
global_results[key] += ret[key]
print(global_results)
def main(type, id, public_id):
maybe_enable_rollbar()
type = type.lower()
if type not in cls_for_type:
print("Error: unknown type '{}'".format(type))
sys.exit(-1)
cls = cls_for_type[type]
if public_id is None and id is None:
print("Error: you should specify an id or public id to query.")
sys.exit(-1)
with global_session_scope() as db_session:
if public_id:
obj = db_session.query(cls).filter(
cls.public_id == public_id).one()
print(obj.id)
elif id:
obj = db_session.query(cls).filter(cls.id == id).one()
print(obj.public_id)
def main(prod, config):
""" Launch the contact search index service. """
level = os.environ.get("LOGLEVEL", inbox_config.get("LOGLEVEL"))
configure_logging(log_level=level)
maybe_enable_rollbar()
if config is not None:
from inbox.util.startup import load_overrides
config_path = os.path.abspath(config)
load_overrides(config_path)
# import here to make sure config overrides are loaded
from inbox.transactions.search import ContactSearchIndexService
if not prod:
preflight()
contact_search_indexer = ContactSearchIndexService()
contact_search_indexer.start()
contact_search_indexer.join()
