def get_repositories():
"""
Returns a list of repositories in the CLA system.
:return: List of repositories in dict format.
:rtype: [dict]
"""
return [repository.to_dict() for repository in get_repository_instance().all()]
def get_repository(repository_id):
"""
Returns the CLA repository requested by ID.
:param repository_id: The repository ID.
:type repository_id: ID
:return: dict representation of the repository object.
:rtype: dict
"""
repository = get_repository_instance()
try:
repository.load(str(repository_id))
except DoesNotExist as err:
return {'errors': {'repository_id': str(err)}}
return repository.to_dict()
def main(
aws_region,
rds_database,
rds_host,
rds_username,
rds_password,
rds_port,
dry_run,
):
"""
This script runs data migration from dynamodb to postgresql
"""
if os.environ.get("STAGE") is None:
logger.warning(
"Please set the 'STAGE' environment varaible - typically one of: {dev, staging, prod}"
)
return
stage = os.environ.get("STAGE")
if dry_run:
exit_cmd = input(
"This is a dry run. "
"You are running the script for the '{}' environment. "
'Press <ENTER> to continue ("exit" to exit): '.format(stage))
else:
exit_cmd = input(
"This is NOT a dry run. "
"You are running the script for the '{}' environment. "
'Press <ENTER> to continue ("exit" to exit): '.format(stage))
if exit_cmd == "exit":
return
start_time = datetime.now()
lock = threading.Lock()
threads = []
count = 0
for _ in range(IMPORT_THREADS):
connection = psycopg2.connect(
database=rds_database,
host=rds_host,
user=rds_username,
password=rds_password,
port=rds_port,
)
cursor = connection.cursor()
worker = Thread(target=import_to_pg,
args=(connection, cursor, tables_queues, dry_run))
worker.start()
threads.append(worker)
# TODO: cla models for signature,company-invitations and user-permissions
try:
user = utils.get_user_instance()
company = utils.get_company_instance()
project = utils.get_project_instance()
repository = utils.get_repository_instance()
github_orgs = utils.get_github_organization_instance()
gerrit = utils.get_gerrit_instance()
#signature = utils.get_signature_instance()
update_queue(user.all())
update_queue(company.all())
update_queue(project.all())
#update_queue(signature.all())
update_queue(repository.all())
update_queue(gerrit.all())
update_queue(github_orgs.all())
# block until all tasks are done
tables_queues.join()
except Exception as err:
logger.error(err)
# End workers
for _ in range(IMPORT_THREADS):
tables_queues.put(None)
for thread in threads:
thread.join()
duration = datetime.now() - start_time
logger.info(
"Data migration to the pg database run for a duration of {} ".format(
duration))