def run_system_metric_monitor(interval=DEFAULT_INTERVAL):
parser = argparse.ArgumentParser(description='Retrieve system metrics i.e cpu utilization, memory & disk usage')
parser.add_argument("crypto_token_file", type=str, help="path to Firebase crypto token file")
args = parser.parse_args()
firebase_client = firebase_util.init_firebase_client(args.crypto_token_file)
runner = threading.Thread(target=get_and_publish_system_metrics, args=(interval,))
runner.start()
reset_flag = sys.argv[3] == '--reset'
replay_flag = sys.argv[3] == '--replay'
if (not reset_flag and not replay_flag):
print(f"Unknown flag: {sys.argv[3]}")
usage()
exit(1)
if len(sys.argv) > 4:
print(f"Unexpected argument {sys.argv[4]}")
usage()
exit(1)
doc_count = 0
skip_count = 0
migration_count = 0
firebase_client = firebase_util.init_firebase_client(crypto_token_path)
# migrate_collection("suggestedReplies", migrate_nook_model.migrate_SuggestedReply)
migrate_collection("nook_conversations",
migrate_nook_model.migrate_Conversation)
migrate_collection("conversationTags", migrate_nook_model.migrate_Tag)
migrate_collection("messageTags", migrate_nook_model.migrate_Tag)
log.info(f"Migration complete")
log.info(f" {migration_count} documents migrated")
log.info(f" {skip_count} documents already migrated")
log.info(f" {doc_count - skip_count - migration_count} documents unchanged")
if migrate_nook_model.warning_count == 0:
log.info(f" no warnings")
else:
log.info(f"")
log.info(f" {migrate_nook_model.warning_count} WARNINGS")
def get_dir_size_in_mb(dirpath):
dir_size = 0
for (path, dirs, files) in os.walk(dirpath):
for file in files:
filename = os.path.join(path, file)
dir_size += os.path.getsize(filename)
return round((dir_size / (1024 * 1024)), 1)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Monitor size of a directory")
parser.add_argument("crypto_token_file",
type=str,
help="path to Firebase crypto token file")
parser.add_argument("project", type=str, help="name of the project")
parser.add_argument("dirpath",
type=str,
help="path of directory to monitor")
args = parser.parse_args()
dirpath = os.path.abspath(args.dirpath)
project = args.project
log = logging.Logger(__file__, args.crypto_token_file)
firebase_client = init_firebase_client(args.crypto_token_file)
while True:
get_size_and_upload(dirpath, project)
time.sleep(DEFAULT_INTERVAL)
if (len(sys.argv) != 4):
print(
"Usage python add_to_firebase.py crypto_token suggested_replies|message_tags|conversation_tags input_path"
)
print(
"Adds data items that don't currently exist, and overwrittes items that already exist."
)
exit(1)
CRYPTO_TOKEN_PATH = sys.argv[1]
CONTENT_TYPE = sys.argv[2]
INPUT_PATH = sys.argv[3]
log = logging.Logger(__file__, CRYPTO_TOKEN_PATH)
firebase_client = firebase_util.init_firebase_client(CRYPTO_TOKEN_PATH)
valid_content_type = [
"suggested_replies", "message_tags", "conversation_tags"
]
if CONTENT_TYPE not in valid_content_type:
log.error("Content type '{}' not known, expected one of {}".format(
CONTENT_TYPE, valid_content_type))
exit(1)
with open(INPUT_PATH, 'r') as f:
data_dict = json.load(f)
if CONTENT_TYPE == "suggested_replies":
# Make sure that the fields that the Firestore model expects exist
# TODO(mariana): Move to using a validator
doc_data["tags_count"] = tags_count
batch.set(doc.reference, doc_data)
batch_count += 1
if batch_count > 450:
log.info(f"Batch about to commit size: {batch_count}")
batch.commit()
batch_count = 0
log.info(f"Batch committed")
batch.commit()
log.info('done migration')
if __name__ == '__main__':
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
epilog=textwrap.dedent("""
Examples:
python migrate_conversation_metrics_model.py <CRYPRO_TOKEN_PATH> Lark_KK-Project-2020-COVID19-KE-URBAN
"""))
parser.add_argument('crypto_token', help='crypto token path')
parser.add_argument('project', help='which project records to migrate')
args = parser.parse_args()
firebase_client = firebase_util.init_firebase_client(args.crypto_token)
log = Logger(__name__)
migrate_project(args.project)
