def parse_and_import_eri_3g4g(parent_dag_name, child_dag_name, start_date,
schedule_interval):
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
t1 = BashOperator(
task_id='check_if_ericsson_bulkcm_raw_files_exist',
# bash_command='if [ 0 -eq `ls -1 /mediation/data/cm/ericsson/3g4g/raw/in | wc -l` ]; then exit 1; fi',
bash_command='echo "0"',
dag=dag)
# Backup previously generate csv files from parsing
t5 = BashOperator(
task_id='backup_ericsson_bulkcm_csv_files',
bash_command=
'mv -f /mediation/data/cm/ericsson/parsed/bulkcm/* /mediation/data/cm/ericsson/parsed/backup/ 2>/dev/null || true',
dag=dag)
t2 = BashOperator(
task_id='run_ericsson_bulkcm_parser',
bash_command=
'java -jar /mediation/bin/boda-bulkcmparser.jar -i /mediation/data/cm/ericsson/raw/bulkcm -o /mediation/data/cm/ericsson/parsed/bulkcm -c /mediation/conf/cm/ericsson_bulkcm_parser.cfg',
dag=dag)
# Truncate ericsson 3g4g cm tables
def clear_ericsson_bulkcm_tables():
bts_utils.truncate_schema_tables(schema="eri_cm_3g4g")
t7 = PythonOperator(task_id='clear_ericsson_bulkcm_tables',
python_callable=clear_ericsson_bulkcm_tables,
dag=dag)
import_csv_files = BashOperator(
task_id='import_ericsson_bulkcm_data',
bash_command=
'python /mediation/bin/load_cm_data_into_db.py ericsson_bulkcm /mediation/data/cm/ericsson/parsed/bulkcm ',
dag=dag)
t_run_ericsson_bulkcm_insert_queries = BashOperator(
task_id='run_ericsson_bulkcm_insert_queries',
bash_command=
'python /mediation/bin/run_cm_load_insert_queries.py ericsson_bulkcm',
dag=dag)
dag.set_dependency('check_if_ericsson_bulkcm_raw_files_exist',
'backup_ericsson_bulkcm_csv_files')
dag.set_dependency('backup_ericsson_bulkcm_csv_files',
'run_ericsson_bulkcm_parser')
dag.set_dependency('run_ericsson_bulkcm_parser',
'clear_ericsson_bulkcm_tables')
dag.set_dependency('clear_ericsson_bulkcm_tables',
'import_ericsson_bulkcm_data')
dag.set_dependency('import_ericsson_bulkcm_data',
'run_ericsson_bulkcm_insert_queries')
return dag
def import_huawei_gexport_parsed_csv(parent_dag_name, child_dag_name,
start_date, schedule_interval):
"""
Parse huawei 2g cm files.
:param parent_dag_name:
:param child_dag_name:
:param start_date:
:param schedule_interval:
:return:
"""
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
t23 = DummyOperator(task_id='branch_huawei_4g_importer', dag=dag)
import_mml_csv = BashOperator(
task_id='import_huawei_4g_mml_data',
bash_command=
'python /mediation/bin/load_cm_data_into_db.py huawei_mml_lte /mediation/data/cm/huawei/parsed/mml_lte ',
dag=dag)
import_nbi_csv = BashOperator(
task_id='import_huawei_4g_nbi_data',
bash_command=
'python /mediation/bin/load_cm_data_into_db.py huawei_nbi_lte /mediation/data/cm/huawei/parsed/nbi_lte ',
dag=dag)
import_nbi_csv = BashOperator(
task_id='import_huawei_4g_gexport_data',
bash_command=
'python /mediation/bin/load_cm_data_into_db.py huawei_gexport_lte /mediation/data/cm/huawei/parsed/gexport_lte ',
dag=dag)
t_join = DummyOperator(
task_id='join_huawei_4g_importer',
dag=dag,
)
dag.set_dependency('branch_huawei_4g_importer',
'import_huawei_4g_mml_data')
dag.set_dependency('branch_huawei_4g_importer',
'import_huawei_4g_nbi_data')
dag.set_dependency('branch_huawei_4g_importer',
'import_huawei_4g_gexport_data')
dag.set_dependency('import_huawei_4g_mml_data', 'join_huawei_4g_importer')
dag.set_dependency('import_huawei_4g_nbi_data', 'join_huawei_4g_importer')
dag.set_dependency('import_huawei_4g_gexport_data',
'join_huawei_4g_importer')
return dag
def parse_and_import_nokia_raml20(parent_dag_name, child_dag_name, start_date, schedule_interval):
"""
Parse and import ZTE xlsx files
:param parent_dag_name:
:param child_dag_name:
:param start_date:
:param schedule_interval:
:return:
"""
dag_id = '%s.%s' % (parent_dag_name, child_dag_name)
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
task_clean_mo_names = BashOperator(
task_id='clean_raml20_mo_names',
bash_command="true",
# r"""
#sed -i "
#/lowerMarginCio/ s//lowerMarginCIO/g;
#/upperMarginCio/ s//upperMarginCIO/g;
#" /mediation/data/cm/nokia/raw/raml20/*.xml || true
# """,
dag=dag
)
parse_nokia_raml20_cm_files = BashOperator(
task_id='parse_nokia_raml20_cm_files',
bash_command='java -jar /mediation/bin/boda-nokiacmdataparser.jar -i /mediation/data/cm/nokia/raw/raml20 -o /mediation/data/cm/nokia/parsed/raml20 -c /mediation/conf/cm/nokia_raml20_parser.cfg',
dag=dag)
import_nokia_raml20_csv = BashOperator(
task_id='import_nokia_raml20_parsed_csv',
bash_command='python /mediation/bin/load_cm_data_into_db.py nokia_raml20 /mediation/data/cm/nokia/parsed/raml20',
dag=dag)
t_run_nokia_raml20_insert_queries = BashOperator(
task_id='run_nokia_raml20_insert_queries',
bash_command='python /mediation/bin/run_cm_load_insert_queries.py nokia_raml20',
dag=dag)
def clear_nokia_raml20_cm_tables():
pass
t50 = PythonOperator(
task_id='clear_nokia_raml20_cm_tables',
python_callable=clear_nokia_raml20_cm_tables,
dag=dag)
dag.set_dependency('clean_raml20_mo_names', 'parse_nokia_raml20_cm_files')
dag.set_dependency('parse_nokia_raml20_cm_files', 'clear_nokia_raml20_cm_tables')
dag.set_dependency('clear_nokia_raml20_cm_tables', 'import_nokia_raml20_parsed_csv')
dag.set_dependency('import_nokia_raml20_parsed_csv', 'run_nokia_raml20_insert_queries')
return dag
def parse_and_import_huawei_4g(parent_dag_name, child_dag_name, start_date,
schedule_interval):
dag_id = '%s.%s' % (parent_dag_name, child_dag_name)
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
t47 = BashOperator(
task_id='check_if_hua_4g_raw_files_exist',
bash_command=
'if [ 0 -eq `ls -1 /mediation/data/cm/huawei/4g/raw/in | wc -l` ]; then exit 1; fi',
dag=dag)
t48 = BashOperator(
task_id='backup_huawei_4g_csv_files',
bash_command=
'mv -f /mediation/data/cm/huawei/4g/parsed/in/* /mediation/data/cm/huawei/4g/parsed/out/ 2>/dev/null || true',
dag=dag)
t49 = BashOperator(
task_id='run_huawei_4g_parser',
bash_command=
'java -jar /mediation/bin/boda-huaweinbixmlparser.jar /mediation/data/cm/huawei/4g/raw/in /mediation/data/cm/huawei/4g/parsed/in /mediation/conf/cm/hua_cm_4g_nbi_parser.cfg',
dag=dag)
# Clear 4G CM data tables
def clear_huawei_4g_cm_tables():
pass
t50 = PythonOperator(task_id='clear_huawei_4g_cm_tables',
python_callable=clear_huawei_4g_cm_tables,
dag=dag)
t51 = BashOperator(
task_id='import_huawei_4g_cm_data',
bash_command=
'export PGPASSWORD=password && psql -h $POSTGRES_HOST -U bodastage -d bts -a -w -f "/mediation/conf/cm/hua_cm_4g_nbi_loader.cfg"',
dag=dag)
dag.set_dependency('check_if_hua_4g_raw_files_exist',
'backup_huawei_4g_csv_files')
dag.set_dependency('backup_huawei_4g_csv_files', 'run_huawei_4g_parser')
dag.set_dependency('run_huawei_4g_parser', 'clear_huawei_4g_cm_tables')
dag.set_dependency('clear_huawei_4g_cm_tables', 'import_huawei_4g_cm_data')
return dag
def parse_and_import_eri_3g4g(parent_dag_name, child_dag_name, start_date,
schedule_interval):
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
t1 = BashOperator(
task_id='check_if_eri_3g4g_raw_files_exist',
bash_command=
'if [ 0 -eq `ls -1 /mediation/data/cm/ericsson/3g4g/raw/in | wc -l` ]; then exit 1; fi',
dag=dag)
# Backup previously generate csv files from parsing
t5 = BashOperator(
task_id='backup_prev_eri_3g4g_csv_files',
bash_command=
'mv -f /mediation/data/cm/ericsson/3g4g/parsed/in/* /mediation/data/cm/ericsson/3g4g/parsed/out/ 2>/dev/null || true',
dag=dag)
t2 = BashOperator(
task_id='run_eri_3g4g_parser',
bash_command=
'java -jar /mediation/bin/boda_bulkcmparser.jar /mediation/data/cm/ericsson/3g4g/raw/in /mediation/data/cm/ericsson/3g4g/parsed/in /mediation/conf/cm/eri_cm_3g4g_parser.cfg',
dag=dag)
# Truncate ericsson 3g4g cm tables
def clear_eri_3g4g_cm_tables():
bts_utils.truncate_schema_tables(schema="eri_cm_3g4g")
t7 = PythonOperator(task_id='clear_eri_3g4g_cm_tables',
python_callable=clear_eri_3g4g_cm_tables,
dag=dag)
# Import csv files into csv files
t3 = BashOperator(
task_id='import_eri_3g4g_cm_data',
bash_command=
'export PGPASSWORD=password && psql -h $POSTGRES_HOST -U bodastage -d bts -a -w -f "/mediation/conf/cm/eri_cm_3g4g_loader.cfg"',
dag=dag)
dag.set_dependency('check_if_eri_3g4g_raw_files_exist',
'backup_prev_eri_3g4g_csv_files')
dag.set_dependency('backup_prev_eri_3g4g_csv_files', 'run_eri_3g4g_parser')
dag.set_dependency('run_eri_3g4g_parser', 'clear_eri_3g4g_cm_tables')
dag.set_dependency('clear_eri_3g4g_cm_tables', 'import_eri_3g4g_cm_data')
return dag
def parse_and_import_eri_2g(parent_dag_name, child_dag_name, start_date,
schedule_interval):
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
t22 = BashOperator(
task_id='check_if_2g_raw_files_exist',
bash_command=
'if [ 0 -eq `ls -1 /mediation/data/cm/ericsson/2g/raw/in | wc -l` ]; then exit 1; fi',
dag=dag)
# .. || true added to make sure the command alwasy succeeds
t21 = BashOperator(
task_id='backup_ericsson_2g_csv_files',
bash_command=
'mv -f /mediation/data/cm/ericsson/2g/parsed/in/* /mediation/data/cm/ericsson/2g/parsed/out/ 2>/dev/null || true',
dag=dag)
t20 = BashOperator(
task_id='run_ericsson_2g_parser',
bash_command=
'java -jar /mediation/bin/boda-ericssoncnaiparser.jar /mediation/data/cm/ericsson/2g/raw/in /mediation/data/cm/ericsson/2g/parsed/in /mediation/conf/cm/eri_cm_2g_cnaiv2_parser.cfg',
dag=dag)
def clear_ericsson_2g_cm_tables():
pass
t19 = PythonOperator(task_id='clear_ericsson_2g_cm_tables',
python_callable=clear_ericsson_2g_cm_tables,
dag=dag)
t18 = BashOperator(
task_id='import_eri_2g_cm_data',
bash_command=
'export PGPASSWORD=password && psql -h $POSTGRES_HOST -U bodastage -d bts -a -w -f "/mediation/conf/cm/eri_cm_2g_loader.cfg"',
dag=dag)
dag.set_dependency('check_if_2g_raw_files_exist',
'backup_ericsson_2g_csv_files')
dag.set_dependency('backup_ericsson_2g_csv_files',
'run_ericsson_2g_parser')
dag.set_dependency('run_ericsson_2g_parser', 'clear_ericsson_2g_cm_tables')
dag.set_dependency('clear_ericsson_2g_cm_tables', 'import_eri_2g_cm_data')
return dag
def run_huawei_2g_parser(parent_dag_name, child_dag_name, start_date, schedule_interval):
"""
Parse huawei 2g cm files.
:param parent_dag_name:
:param child_dag_name:
:param start_date:
:param schedule_interval:
:return:
"""
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
def get_cm_file_format():
# if 'huawei_mml'
return 'run_huawei_2g_mml_parser'
t23 = BranchPythonOperator(
task_id='branch_huawei_2g_parser',
python_callable=get_cm_file_format,
dag=dag)
t29 = BashOperator(
task_id='run_huawei_2g_xml_nbi_parser',
bash_command='java -jar /mediation/bin/boda-huaweinbixmlparser.jar /mediation/data/cm/huawei/2g/raw/in /mediation/data/cm/huawei/2g/parsed/in /mediation/conf/cm/hua_cm_2g_nbi_parameters.cfg',
dag=dag)
t29_2 = BashOperator(
task_id='run_huawei_2g_mml_parser',
bash_command='java -jar /mediation/bin/boda-huaweimmlparser.jar /mediation/data/cm/huawei/2g/raw/in /mediation/data/cm/huawei/2g/parsed/in /mediation/conf/cm/hua_cm_2g_mml_parser.cfg',
dag=dag)
t_join = DummyOperator(
task_id='join_huawei_2g_parser',
dag=dag,
)
dag.set_dependency('branch_huawei_2g_parser', 'run_huawei_2g_mml_parser')
dag.set_dependency('branch_huawei_2g_parser', 'run_huawei_2g_xml_nbi_parser')
dag.set_dependency('run_huawei_2g_mml_parser', 'join_huawei_2g_parser')
dag.set_dependency('run_huawei_2g_xml_nbi_parser', 'join_huawei_2g_parser')
return dag
def parse_and_import_huawei_cfgsyn(parent_dag_name, child_dag_name, start_date,
schedule_interval):
dag_id = '%s.%s' % (parent_dag_name, child_dag_name)
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
parse_huawei_cfgsyn_cm_files = BashOperator(
task_id='parse_huawei_cfgsyn_cm_files',
bash_command=
'java -jar /mediation/bin/boda-huaweicfgsynparser.jar /mediation/data/cm/huawei/raw/cfgsyn /mediation/data/cm/huawei/parsed/cfgsyn /mediation/conf/cm/huawei_cfgsyn_parser.cfg',
dag=dag)
import_nbi_csv = BashOperator(
task_id='import_huawei_cfgsyn_parsed_csv',
bash_command=
'python /mediation/bin/load_cm_data_into_db.py huawei_cfgsyn /mediation/data/cm/huawei/parsed/cfgsyn',
dag=dag)
# Clear cfgsyn CM data tables
def clear_huawei_cfgsyn_cm_tables():
pass
t50 = PythonOperator(task_id='clear_huawei_cfgsyn_cm_tables',
python_callable=clear_huawei_cfgsyn_cm_tables,
dag=dag)
dag.set_dependency('parse_huawei_cfgsyn_cm_files',
'clear_huawei_cfgsyn_cm_tables')
dag.set_dependency('clear_huawei_cfgsyn_cm_tables',
'import_huawei_cfgsyn_parsed_csv')
return dag
def parse_and_import_eri_2g(parent_dag_name, child_dag_name, start_date, schedule_interval):
"""Parse and import Ericsson GSM CM data"""
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
t22 = BashOperator(
task_id='check_if_cnaiv2_raw_files_exist',
# bash_command='if [ 0 -eq `ls -1 /mediation/data/cm/ericsson/2g/raw/in | wc -l` ]; then exit 1; fi',
bash_command='echo 0;',
dag=dag)
# .. || true added to make sure the command alwasy succeeds
t21 = BashOperator(
task_id='backup_ericsson_cnaiv2_csv_files',
bash_command='mv -f /mediation/data/cm/ericsson/parsed/cnaiv2/* /mediation/data/cm/ericsson/parsed/backup/ 2>/dev/null || true',
dag=dag)
t20 = BashOperator(
task_id='run_ericsson_cnaiv2_parser',
bash_command='java -jar /mediation/bin/boda-ericssoncnaiparser.jar /mediation/data/cm/ericsson/raw/cnaiv2 /mediation/data/cm/ericsson/parsed/cnaiv2 /mediation/conf/cm/ericsson_cnaiv2_gsm_parser.cfg',
dag=dag)
def clear_ericsson_cnaiv2_cm_tables():
pass
t19 = PythonOperator(
task_id='clear_ericsson_cnaiv2_cm_tables',
python_callable=clear_ericsson_cnaiv2_cm_tables,
dag=dag)
import_csv_files = BashOperator(
task_id='import_ericsson_cnaiv2_data',
bash_command='python /mediation/bin/load_cm_data_into_db.py ericsson_cnaiv2 /mediation/data/cm/ericsson/parsed/cnaiv2 ',
dag=dag)
dag.set_dependency('check_if_cnaiv2_raw_files_exist', 'backup_ericsson_cnaiv2_csv_files')
dag.set_dependency('backup_ericsson_cnaiv2_csv_files', 'run_ericsson_cnaiv2_parser')
dag.set_dependency('run_ericsson_cnaiv2_parser', 'clear_ericsson_cnaiv2_cm_tables')
dag.set_dependency('clear_ericsson_cnaiv2_cm_tables', 'import_ericsson_cnaiv2_data')
return dag
def parse_and_import_huawei_3g(parent_dag_name, child_dag_name, start_date,
schedule_interval):
dag_id = '%s.%s' % (parent_dag_name, child_dag_name)
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
t42 = BashOperator(
task_id='check_if_hua_3g_raw_files_exist',
bash_command=
'if [ 0 -eq `ls -1 /mediation/data/cm/huawei/3g/raw/in | wc -l` ]; then exit 1; fi',
dag=dag)
t43 = BashOperator(
task_id='backup_huawei_3g_csv_files',
bash_command=
'mv -f /mediation/data/cm/huawei/3g/parsed/in/* /mediation/data/cm/huawei/3g/parsed/out/ 2>/dev/null || true',
dag=dag)
# Clear 3G CM data tables
def clear_huawei_3g_cm_tables():
pass
t45 = PythonOperator(task_id='clear_huawei_3g_cm_tables',
python_callable=clear_huawei_3g_cm_tables,
dag=dag)
t46 = BashOperator(
task_id='import_huawei_3g_cm_data',
bash_command=
'export PGPASSWORD=password && psql -h $POSTGRES_HOST -U bodastage -d bts -a -w -f "/mediation/conf/cm/hua_cm_3g_nbi_loader.cfg"',
dag=dag)
sub_dag_parser_huawei_3g_cm_files = SubDagOperator(
subdag=run_huawei_3g_parser(dag_id,
'parse_huawei_3g_cm_files',
start_date=dag.start_date,
schedule_interval=dag.schedule_interval),
task_id='parse_huawei_3g_cm_files',
dag=dag,
)
dag.set_dependency('check_if_hua_3g_raw_files_exist',
'backup_huawei_3g_csv_files')
dag.set_dependency('backup_huawei_3g_csv_files',
'parse_huawei_3g_cm_files')
dag.set_dependency('parse_huawei_3g_cm_files', 'clear_huawei_3g_cm_tables')
dag.set_dependency('clear_huawei_3g_cm_tables', 'import_huawei_3g_cm_data')
return dag
def parse_and_import_zte_excel(parent_dag_name, child_dag_name, start_date,
schedule_interval):
"""
Parse and import ZTE xlsx files
:param parent_dag_name:
:param child_dag_name:
:param start_date:
:param schedule_interval:
:return:
"""
dag_id = '%s.%s' % (parent_dag_name, child_dag_name)
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
parse_zte_excel_cm_files = BashOperator(
task_id='parse_zte_excel_cm_files',
bash_command=
'python /mediation/bin/parse_zte_excel.py -i /mediation/data/cm/zte/raw/excel -o /mediation/data/cm/zte/parsed/excel -c /mediation/conf/cm/zte_excel_parser.cfg',
dag=dag)
import_zte_excel_csv = BashOperator(
task_id='import_zte_excel_parsed_csv',
bash_command=
'python /mediation/bin/load_cm_data_into_db.py zte_excel /mediation/data/cm/zte/parsed/excel',
dag=dag)
t_run_zte_excel_insert_queries = BashOperator(
task_id='run_zte_excel_insert_queries',
bash_command=
'python /mediation/bin/run_cm_load_insert_queries.py zte_excel',
dag=dag)
# Clear 4G CM data tables
def clear_zte_excel_cm_tables():
pass
t50 = PythonOperator(task_id='clear_zte_excel_cm_tables',
python_callable=clear_zte_excel_cm_tables,
dag=dag)
dag.set_dependency('parse_zte_excel_cm_files', 'clear_zte_excel_cm_tables')
dag.set_dependency('clear_zte_excel_cm_tables',
'import_zte_excel_parsed_csv')
dag.set_dependency('import_zte_excel_parsed_csv',
'run_zte_excel_insert_queries')
return dag
def parse_and_import_zte_2g(parent_dag_name, child_dag_name, start_date,
schedule_interval):
"""Parse and import ZTE 2G"""
dag_id = '%s.%s' % (parent_dag_name, child_dag_name)
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
# @TODO: Investigate other ways to check if there are not files yet
t28 = BashOperator(
task_id='check_if_zte_2g_raw_files_exist',
bash_command='ls -1 /mediation/data/cm/zte/raw/in | wc -l',
dag=dag)
# @TODO: Backup parsed files
t30 = BashOperator(
task_id='backup_zte_2g_csv_files',
bash_command=
'mv -f /mediation/data/cm/zte/parsed/bulkcm_gsm/* /mediation/data/cm/zte/parsed/backup 2>/dev/null || true',
dag=dag)
def clear_zte_2g_cm_tables():
pass
t31 = PythonOperator(task_id='clear_zte_2g_cm_tables',
python_callable=clear_zte_2g_cm_tables,
dag=dag)
parse_zte_2g_cm_files = BashOperator(
task_id='parse_zte_2g_cm_files',
bash_command=
'java -jar /mediation/bin/boda-bulkcmparser.jar /mediation/data/cm/zte/raw/bulkcm_gsm /mediation/data/cm/zte/parsed/bulkcm_gsm /mediation/conf/cm/zte_cm_2g_blkcm_parser.cfg',
dag=dag)
import_zte_cm_data = BashOperator(
task_id='import_zte_2g_cm_data',
bash_command=
'python /mediation/bin/load_cm_data_into_db.py zte_bulkcm_gsm /mediation/data/cm/zte/parsed/bulkcm_gsm ',
dag=dag)
dag.set_dependency('check_if_zte_2g_raw_files_exist',
'backup_zte_2g_csv_files')
dag.set_dependency('backup_zte_2g_csv_files', 'parse_zte_2g_cm_files')
dag.set_dependency('parse_zte_2g_cm_files', 'clear_zte_2g_cm_tables')
dag.set_dependency('clear_zte_2g_cm_tables', 'import_zte_2g_cm_data')
return dag
def parse_and_import_zte_bulkcm(parent_dag_name, child_dag_name, start_date,
schedule_interval):
dag_id = '%s.%s' % (parent_dag_name, child_dag_name)
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
parse_zte_bulkcm_cm_files = BashOperator(
task_id='parse_zte_bulkcm_cm_files',
bash_command=
'java -jar /mediation/bin/boda-bulkcmparser.jar -i /mediation/data/cm/zte/raw/bulkcm -o /mediation/data/cm/zte/parsed/bulkcm -c /mediation/conf/cm/zte_bulkcm_parser.cfg',
dag=dag)
import_mml_csv = BashOperator(
task_id='import_zte_bulkcm_parsed_csv',
bash_command=
'python /mediation/bin/load_cm_data_into_db.py zte_bulkcm /mediation/data/cm/zte/parsed/bulkcm',
dag=dag)
t_run_zte_bulkcm_insert_queries = BashOperator(
task_id='run_zte_bulkcm_insert_queries',
bash_command=
'python /mediation/bin/run_cm_load_insert_queries.py zte_bulkcm',
dag=dag)
# Clear 4G CM data tables
def clear_zte_bulkcm_cm_tables():
pass
t50 = PythonOperator(task_id='clear_zte_bulkcm_cm_tables',
python_callable=clear_zte_bulkcm_cm_tables,
dag=dag)
dag.set_dependency('parse_zte_bulkcm_cm_files',
'clear_zte_bulkcm_cm_tables')
dag.set_dependency('clear_zte_bulkcm_cm_tables',
'import_zte_bulkcm_parsed_csv')
dag.set_dependency('import_zte_bulkcm_parsed_csv',
'run_zte_bulkcm_insert_queries')
return dag
def parse_and_import_huawei_rnp(parent_dag_name, child_dag_name, start_date, schedule_interval):
dag_id = '%s.%s' % (parent_dag_name, child_dag_name)
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
parse_huawei_rnp_cm_files = BashOperator(
task_id='parse_huawei_rnp_cm_files',
bash_command='python /mediation/bin/parse_huawei_rnp_data_templates.py /mediation/data/cm/huawei/raw/rnp /mediation/data/cm/huawei/parsed/rnp /mediation/conf/cm/huawei_rnp_parser.cfg',
dag=dag)
import_nbi_csv = BashOperator(
task_id='import_huawei_rnp_parsed_csv',
bash_command='python /mediation/bin/load_cm_data_into_db.py huawei_rnp /mediation/data/cm/huawei/parsed/rnp',
dag=dag)
t_run_huawei_gexport_insert_queries = BashOperator(
task_id='run_huawei_rnp_insert_queries',
bash_command='python /mediation/bin/run_cm_load_insert_queries.py huawei_rnp',
dag=dag)
# Clear 4G CM data tables
def clear_huawei_rnp_cm_tables():
pass
t50 = PythonOperator(
task_id='clear_huawei_rnp_cm_tables',
python_callable=clear_huawei_rnp_cm_tables,
dag=dag)
dag.set_dependency('parse_huawei_rnp_cm_files', 'clear_huawei_rnp_cm_tables')
dag.set_dependency('clear_huawei_rnp_cm_tables', 'import_huawei_rnp_parsed_csv')
dag.set_dependency('import_huawei_rnp_parsed_csv', 'run_huawei_rnp_insert_queries')
return dag
def parse_and_import_huawei_3g(parent_dag_name, child_dag_name, start_date,
schedule_interval):
dag_id = '%s.%s' % (parent_dag_name, child_dag_name)
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
# @TODO: Investigate other ways to check if there are not files yet considering all the formats supported
t42 = BashOperator(
task_id='check_if_hua_3g_raw_files_exist',
# bash_command='if [ 0 -eq `ls -1 /mediation/data/cm/huawei/3g/raw/in | wc -l` ]; then exit 1; fi',
bash_command='ls -1 /mediation/data/cm/huawei/2g/raw/in | wc -l',
dag=dag)
t43 = BashOperator(
task_id='backup_huawei_3g_csv_files',
bash_command=
'mv -f /mediation/data/cm/huawei/parsed/{nbi_umts,nbi_sran,nbi_lte,nbi_gsm,mml_umts,mml_lte,mml_gsm,gexport_wcdma,gexport_sran,gexport_other,gexport_lte,gexport_gsm,gexport_cdma}/* /mediation/data/cm/huawei/parsed/backup 2>/dev/null || true',
dag=dag)
# Clear 3G CM data tables
def clear_huawei_3g_cm_tables():
pass
t45 = PythonOperator(task_id='clear_huawei_3g_cm_tables',
python_callable=clear_huawei_3g_cm_tables,
dag=dag)
# t46 = BashOperator(
# task_id='import_huawei_3g_cm_data',
# bash_command='export PGPASSWORD=password && psql -h $POSTGRES_HOST -U bodastage -d bts -a -w -f "/mediation/conf/cm/hua_cm_3g_nbi_loader.cfg"',
# dag=dag)
sub_dag_import_huawei_3g_csv = SubDagOperator(
subdag=import_huawei_3g_parsed_csv(
dag_id,
'import_huawei_3g_parsed_csv',
start_date=dag.start_date,
schedule_interval=dag.schedule_interval),
task_id='import_huawei_3g_parsed_csv',
dag=dag,
)
sub_dag_parser_huawei_3g_cm_files = SubDagOperator(
subdag=run_huawei_3g_parser(dag_id,
'parse_huawei_3g_cm_files',
start_date=dag.start_date,
schedule_interval=dag.schedule_interval),
task_id='parse_huawei_3g_cm_files',
dag=dag,
)
dag.set_dependency('check_if_hua_3g_raw_files_exist',
'backup_huawei_3g_csv_files')
dag.set_dependency('backup_huawei_3g_csv_files',
'parse_huawei_3g_cm_files')
dag.set_dependency('parse_huawei_3g_cm_files', 'clear_huawei_3g_cm_tables')
dag.set_dependency('clear_huawei_3g_cm_tables',
'import_huawei_3g_parsed_csv')
return dag
def extract_2g_externals(parent_dag_name, child_dag_name, start_date,
schedule_interval):
"""
Parse huawei 4g cm files.
:param parent_dag_name:
:param child_dag_name:
:param start_date:
:param schedule_interval:
:return:
"""
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
branch_start = BranchPythonOperator(task_id='branch_huawei_2g_externals',
python_callable=pre_clean_up,
dag=dag)
def extract_huawei_2g_externals():
huawei_cm.extract_live_network_2g_externals_on_2g()
huawei_cm.extract_live_network_3g_externals_on_2g()
huawei_cm.extract_live_network_4g_externals_on_2g()
t66 = PythonOperator(task_id='extract_huawei_2g_externals',
python_callable=extract_huawei_2g_externals,
dag=dag)
t29 = BashOperator(
task_id='run_huawei_4g_xml_nbi_parser',
bash_command=
'java -jar /mediation/bin/boda-huaweinbixmlparser.jar /mediation/data/cm/huawei/raw/nbi_lte /mediation/data/cm/huawei/parsed/nbi_lte /mediation/conf/cm/huawei_nbi_lte_parser.cfg',
dag=dag)
t29_2 = BashOperator(
task_id='run_huawei_4g_mml_parser',
bash_command=
'java -jar /mediation/bin/boda-huaweimmlparser.jar /mediation/data/cm/huawei/raw/mml_lte /mediation/data/cm/huawei/parsed/mml_lte /mediation/conf/cm/huawei_mml_lte_parser.cfg',
dag=dag)
run_huawei_2g_xml_gexport_parser = BashOperator(
task_id='run_huawei_4g_xml_gexport_parser',
bash_command=
'java -jar /mediation/bin/boda-huaweicmobjectparser.jar /mediation/data/cm/huawei/raw/gexport_lte /mediation/data/cm/huawei/parsed/gexport_lte /mediation/conf/cm/huawei_gexport_lte_parser.cfg',
dag=dag)
t_join = DummyOperator(
task_id='join_huawei_2g_externals',
dag=dag,
)
dag.set_dependency('branch_huawei_4g_parser', 'run_huawei_4g_mml_parser')
dag.set_dependency('branch_huawei_4g_parser',
'run_huawei_4g_xml_nbi_parser')
dag.set_dependency('branch_huawei_4g_parser',
'run_huawei_4g_xml_gexport_parser')
dag.set_dependency('run_huawei_4g_mml_parser', 'join_huawei_4g_parser')
dag.set_dependency('run_huawei_4g_xml_nbi_parser', 'join_huawei_4g_parser')
dag.set_dependency('run_huawei_4g_xml_gexport_parser',
'join_huawei_4g_parser')
return dag
def run_house_keeping_tasks(parent_dag_name, child_dag_name, start_date,
schedule_interval):
"""
Live network external definitions for all vendors
:param parent_dag_name:
:param child_dag_name:
:param start_date:
:param schedule_interval:
:return:
"""
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
start_house_keeping_tasks = DummyOperator(task_id='start_house_keeping',
dag=dag)
join_externals_task = DummyOperator(task_id='end_house_keeping', dag=dag)
backup_huawei_cm_dumps_task = BashOperator(
task_id='backup_huawei_cm_dumps',
bash_command=
'mv -f /mediation/data/cm/huawei/raw/{nbi,mml,gexport,cfgsyn}/* /mediation/data/cm/huawei/raw/backup/ 2>/dev/null || true',
dag=dag)
backup_ericsson_cm_dumps_task = BashOperator(
task_id='backup_ericsson_cm_dumps',
bash_command=
'mv -f /mediation/data/cm/ericsson/raw/{bulkcm,eaw,cnaiv2}/* /mediation/data/cm/ericsson/raw/backup/ 2>/dev/null || true',
dag=dag)
backup_zte_cm_dumps_task = BashOperator(
task_id='backup_zte_cm_dumps',
bash_command=
'mv -f /mediation/data/cm/zte/raw/{bulkcm,excel}/* /mediation/data/cm/zte/raw/backup/ 2>/dev/null || true',
dag=dag)
backup_nokia_cm_dumps_task = BashOperator(
task_id='backup_nokia_cm_dumps',
bash_command=
'mv -f /mediation/data/cm/nokia/raw/{raml20}/* /mediation/data/cm/nokia/raw/backup/ 2>/dev/null || true',
dag=dag)
dag.set_dependency('start_house_keeping', 'backup_huawei_cm_dumps')
dag.set_dependency('start_house_keeping', 'backup_ericsson_cm_dumps')
dag.set_dependency('start_house_keeping', 'backup_zte_cm_dumps')
dag.set_dependency('start_house_keeping', 'backup_nokia_cm_dumps')
dag.set_dependency('backup_huawei_cm_dumps', 'end_house_keeping')
dag.set_dependency('backup_ericsson_cm_dumps', 'end_house_keeping')
dag.set_dependency('backup_zte_cm_dumps', 'end_house_keeping')
dag.set_dependency('backup_nokia_cm_dumps', 'end_house_keeping')
return dag
def extract_network_externals(parent_dag_name, child_dag_name, start_date, schedule_interval):
"""
Live network external definitions for all vendors
:param parent_dag_name:
:param child_dag_name:
:param start_date:
:param schedule_interval:
:return:
"""
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
branch_externals_task = DummyOperator( task_id='branch_externals', dag=dag)
join_externals_task = DummyOperator(task_id='join_externals', dag=dag)
def extract_external_definitions_on_ericsson():
ericsson_cm.extract_live_network_externals_on_2g()
ericsson_cm.extract_live_network_externals_on_3g()
ericsson_cm.extract_live_network_externals_on_4g()
extract_external_definitions_on_ericsson_task = BranchPythonOperator(
task_id='extract_external_definitions_on_ericsson',
python_callable=extract_external_definitions_on_ericsson,
dag=dag)
def extract_external_definitions_on_huawei():
huawei_cm.extract_live_network_externals_on_2g()
huawei_cm.extract_live_network_externals_on_3g()
huawei_cm.extract_live_network_externals_on_4g()
extract_external_definitions_on_huawei_task = BranchPythonOperator(
task_id='extract_external_definitions_on_huawei',
python_callable=extract_external_definitions_on_huawei,
dag=dag)
def extract_external_definitions_on_zte():
zte_cm.extract_live_network_externals_on_2g()
zte_cm.extract_live_network_externals_on_3g()
zte_cm.extract_live_network_externals_on_4g()
extract_external_definitions_on_zte_task = BranchPythonOperator(
task_id='extract_external_definitions_on_zte',
python_callable=extract_external_definitions_on_zte,
dag=dag)
dag.set_dependency('branch_externals', 'extract_external_definitions_on_ericsson')
dag.set_dependency('branch_externals', 'extract_external_definitions_on_huawei')
dag.set_dependency('branch_externals', 'extract_external_definitions_on_zte')
dag.set_dependency('extract_external_definitions_on_ericsson', 'join_externals')
dag.set_dependency('extract_external_definitions_on_huawei', 'join_externals')
dag.set_dependency('extract_external_definitions_on_zte', 'join_externals')
return dag
deps = {
'task_1':[
'wf_upstream_1',
'wf_upstream_2',
],
'task_2':[
'wf_upstream_1',
'wf_upstream_2',
'wf_upstream_3',
],
}
# Explicitly define the dependencies in the DAG
for downstream, upstream_list in deps.iteritems():
for upstream in upstream_list:
dag.set_dependency(upstream, downstream)
'''
Principles of a Good ETL Pipeline:
- Partition data tables
Data partitioning can be especially useful
when dealing with large-size tables with a
long history. When data is partitioned using
datestamps, we can leverage dynamic partions
to parallelize backfilling.
CREATE TABLE IF NOT EXISTS fct_bookings(
id_listing BIGINT COMMENT 'Unique ID of the listing'
, id_host BIGINT COMMENT 'Unique ID of the host who owns the listing'
, m_bookings BIGINT COMMENT 'Denoted 1 if a booking transaction occurred'
)
def run_network_baseline_audit():
nb.run_baseline_audit()
network_baseline_audit = PythonOperator(
task_id='run_network_baseline_audit',
python_callable=run_network_baseline_audit,
dag=dag)
ext_dep = ExternalTaskSensor(external_dag_id='cm_load',
external_task_id='end_cm_load',
task_id='start_baseline',
dag=dag)
dag.set_dependency('start_baseline', 'huawei_2g_value_counts')
dag.set_dependency('start_baseline', 'huawei_3g_value_counts')
dag.set_dependency('start_baseline', 'huawei_4g_value_counts')
dag.set_dependency('start_baseline', 'ericsson_2g_value_counts')
dag.set_dependency('start_baseline', 'ericsson_3g_value_counts')
dag.set_dependency('start_baseline', 'ericsson_4g_value_counts')
dag.set_dependency('start_baseline', 'zte_2g_value_counts')
dag.set_dependency('start_baseline', 'zte_3g_value_counts')
dag.set_dependency('start_baseline', 'zte_4g_value_counts')
dag.set_dependency('huawei_2g_value_counts', 'compute_network_baseline')
dag.set_dependency('huawei_3g_value_counts', 'compute_network_baseline')
dag.set_dependency('huawei_4g_value_counts', 'compute_network_baseline')
def parse_and_import_huawei_2g(parent_dag_name, child_dag_name, start_date,
schedule_interval):
dag_id = '%s.%s' % (parent_dag_name, child_dag_name)
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
# @TODO: Investigate other ways to check if there are not files yet
t28 = BashOperator(
task_id='check_if_huawei_2g_raw_files_exist',
# bash_command='if [ 0 -eq `ls -1 /mediation/data/cm/huawei/2g/raw/in | wc -l` ]; then exit 1; fi',
bash_command='ls -1 /mediation/data/cm/huawei/2g/raw/in | wc -l',
dag=dag)
# @TODO: Backup parsed files
t30 = BashOperator(
task_id='backup_huawei_2g_csv_files',
# bash_command='mv -f /mediation/data/cm/huawei/2g/parsed/in/* /mediation/data/cm/huawei/2g/parsed/out/ 2>/dev/null || true',
bash_command=
'mv -f /mediation/data/cm/huawei/2g/parsed/in/* /mediation/data/cm/huawei/2g/parsed/out/ 2>/dev/null || true',
dag=dag)
def clear_huawei_2g_cm_tables():
pass
t31 = PythonOperator(task_id='clear_huawei_2g_cm_tables',
python_callable=clear_huawei_2g_cm_tables,
dag=dag)
# Run Huawei 2G parser
sub_dag_parse_huawei_2g_cm_files = SubDagOperator(
subdag=run_huawei_2g_parser(dag_id,
'parse_huawei_2g_cm_files',
start_date=dag.start_date,
schedule_interval=dag.schedule_interval),
task_id='parse_huawei_2g_cm_files',
dag=dag,
)
sub_dag_import_huawei_2g_csv = SubDagOperator(
subdag=import_huawei_2g_parsed_csv(
dag_id,
'import_huawei_2g_parsed_csv',
start_date=dag.start_date,
schedule_interval=dag.schedule_interval),
task_id='import_huawei_2g_parsed_csv',
dag=dag,
)
#
# t32 = BashOperator(
# task_id='import_huawei_2g_cm_data',
# bash_command='export PGPASSWORD=password && psql -h $POSTGRES_HOST -U bodastage -d bts -a -w -f "/mediation/conf/cm/hua_cm_2g_nbi_loader.cfg"',
# dag=dag)
dag.set_dependency('check_if_huawei_2g_raw_files_exist',
'backup_huawei_2g_csv_files')
dag.set_dependency('backup_huawei_2g_csv_files',
'parse_huawei_2g_cm_files')
dag.set_dependency('parse_huawei_2g_cm_files', 'clear_huawei_2g_cm_tables')
dag.set_dependency('clear_huawei_2g_cm_tables',
'import_huawei_2g_parsed_csv')
return dag
def run_huawei_4g_parser(parent_dag_name, child_dag_name, start_date, schedule_interval):
"""
Parse huawei 4g cm files.
:param parent_dag_name:
:param child_dag_name:
:param start_date:
:param schedule_interval:
:return:
"""
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
def pre_clean_up():
pass
t23 = DummyOperator(
task_id='branch_huawei_4g_parser',
dag=dag)
t29 = BashOperator(
task_id='run_huawei_4g_xml_nbi_parser',
bash_command='java -jar /mediation/bin/boda-huaweinbixmlparser.jar /mediation/data/cm/huawei/raw/nbi_lte /mediation/data/cm/huawei/parsed/nbi_lte /mediation/conf/cm/huawei_nbi_lte_parser.cfg',
dag=dag)
t29_2 = BashOperator(
task_id='run_huawei_4g_mml_parser',
bash_command='java -jar /mediation/bin/boda-huaweimmlparser.jar /mediation/data/cm/huawei/raw/mml_lte /mediation/data/cm/huawei/parsed/mml_lte /mediation/conf/cm/huawei_mml_lte_parser.cfg',
dag=dag)
run_huawei_2g_xml_gexport_parser = BashOperator(
task_id='run_huawei_4g_xml_gexport_parser',
bash_command='java -jar /mediation/bin/boda-huaweicmobjectparser.jar /mediation/data/cm/huawei/raw/gexport_lte /mediation/data/cm/huawei/parsed/gexport_lte /mediation/conf/cm/huawei_gexport_lte_parser.cfg',
dag=dag)
t_join = DummyOperator(
task_id='join_huawei_4g_parser',
dag=dag,
)
dag.set_dependency('branch_huawei_4g_parser', 'run_huawei_4g_mml_parser')
dag.set_dependency('branch_huawei_4g_parser', 'run_huawei_4g_xml_nbi_parser')
dag.set_dependency('branch_huawei_4g_parser', 'run_huawei_4g_xml_gexport_parser')
dag.set_dependency('run_huawei_4g_mml_parser', 'join_huawei_4g_parser')
dag.set_dependency('run_huawei_4g_xml_nbi_parser', 'join_huawei_4g_parser')
dag.set_dependency('run_huawei_4g_xml_gexport_parser', 'join_huawei_4g_parser')
return dag
inconsistent_lte_externals_task = PythonOperator(
task_id='generate_inconsistent_lte_externals',
python_callable=inconsistent_lte_externals,
dag=dag)
generate_missing_one_way_relations_task = PythonOperator(
task_id='generate_missing_one_way_relations',
python_callable=generate_missing_one_way_relations,
dag=dag)
generate_missing_cosite_relations_task = PythonOperator(
task_id='generate_missing_cosite_relations',
python_callable=generate_missing_cosite_relations,
dag=dag)
dag.set_dependency('start_network_audits',
'generate_inconsistent_gsm_externals')
dag.set_dependency('start_network_audits',
'generate_inconsistent_umts_externals')
dag.set_dependency('start_network_audits',
'generate_inconsistent_lte_externals')
dag.set_dependency('start_network_audits',
'generate_missing_one_way_relations')
dag.set_dependency('start_network_audits', 'generate_missing_cosite_relations')
dag.set_dependency('generate_inconsistent_gsm_externals', 'end_network_audits')
dag.set_dependency('generate_inconsistent_umts_externals',
'end_network_audits')
dag.set_dependency('generate_inconsistent_lte_externals', 'end_network_audits')
dag.set_dependency('generate_missing_one_way_relations', 'end_network_audits')
dag.set_dependency('generate_missing_cosite_relations', 'end_network_audits')
def run_huawei_2g_parser(parent_dag_name, child_dag_name, start_date,
schedule_interval):
"""
Parse huawei 2g cm files.
:param parent_dag_name:
:param child_dag_name:
:param start_date:
:param schedule_interval:
:return:
"""
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
t23 = DummyOperator(task_id='branch_huawei_2g_parser', dag=dag)
t29 = BashOperator(
task_id='run_huawei_2g_xml_nbi_parser',
bash_command=
'java -jar /mediation/bin/boda-huaweinbixmlparser.jar /mediation/data/cm/huawei/raw/nbi_gsm /mediation/data/cm/huawei/parsed/nbi_gsm /mediation/conf/cm/huawei_nbi_gsm_parser.cfg',
dag=dag)
t29_2 = BashOperator(
task_id='run_huawei_2g_mml_parser',
bash_command=
'java -jar /mediation/bin/boda-huaweimmlparser.jar /mediation/data/cm/huawei/raw/mml_gsm /mediation/data/cm/huawei/parsed/mml_gsm /mediation/conf/cm/huawei_mml_gsm_parser.cfg',
dag=dag)
t29_3 = BashOperator(
task_id='run_huawei_2g_xml_gexport_parser',
bash_command=
'java -jar /mediation/bin/boda-huaweicmobjectparser.jar /mediation/data/cm/huawei/raw/gexport_gsm /mediation/data/cm/huawei/parsed/gexport_gsm /mediation/conf/cm/huawei_gexport_gsm_parser.cfg',
dag=dag)
t_join = DummyOperator(
task_id='join_huawei_2g_parser',
dag=dag,
)
t_strip_neversion_from_mo = BashOperator(
task_id='strip_neversion_from_gsm_mos',
bash_command=
'sed -i -r " s/_(BSC6900GSM|BSC6900UMTS|BSC6900GU|BSC6910GSM|BSC6910UMTS|BSC6910GU)//ig; s/_(BTS3900|PICOBTS3900|BTS3911B|PICOBTS3911B|MICROBTS3900|MICROBTS3911B)//ig;" /mediation/data/cm/huawei/raw/gexport_gsm/*',
dag=dag)
dag.set_dependency('branch_huawei_2g_parser', 'run_huawei_2g_mml_parser')
dag.set_dependency('branch_huawei_2g_parser',
'run_huawei_2g_xml_nbi_parser')
dag.set_dependency('branch_huawei_2g_parser',
'strip_neversion_from_gsm_mos')
dag.set_dependency('strip_neversion_from_gsm_mos',
'run_huawei_2g_xml_gexport_parser')
dag.set_dependency('run_huawei_2g_mml_parser', 'join_huawei_2g_parser')
dag.set_dependency('run_huawei_2g_xml_nbi_parser', 'join_huawei_2g_parser')
dag.set_dependency('run_huawei_2g_xml_gexport_parser',
'join_huawei_2g_parser')
return dag
trigger_rule=TriggerRule.ALL_SUCCESS,
dag=dag)
t81 = DummyOperator(task_id='join_ericsson_supported',
trigger_rule=TriggerRule.ONE_SUCCESS,
dag=dag)
t82 = DummyOperator(task_id='join_huawei_supported',
trigger_rule=TriggerRule.ONE_SUCCESS,
dag=dag)
# Ericsson
# Build dependency graph
# dag.set_dependency('start_cm_etlp','is_ericsson_supported')
dag.set_dependency('start_cm_etlp', 'is_ericsson_2g_supported')
dag.set_dependency('start_cm_etlp', 'is_ericsson_3g4g_supported')
dag.set_dependency('is_ericsson_3g4g_supported', 'ericsson_3g4g_supported')
dag.set_dependency('is_ericsson_3g4g_supported', 'ericsson_3g4g_not_supported')
# dag.set_dependency('is_ericsson_supported','ericsson_is_supported')
# dag.set_dependency('is_ericsson_supported','ericsson_not_supported')
# dag.set_dependency('ericsson_not_supported','join_ericsson_supported')
dag.set_dependency('ericsson_cm_done', 'join_ericsson_supported')
dag.set_dependency('join_ericsson_supported', 'end_cm_etlp')
dag.set_dependency('ericsson_3g4g_not_supported', 'ericsson_cm_done')
dag.set_dependency('ericsson_3g4g_supported',
'parser_and_import_ericsson_3g4g')
def parse_and_import_huawei_gexport(parent_dag_name, child_dag_name,
start_date, schedule_interval):
dag_id = '%s.%s' % (parent_dag_name, child_dag_name)
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
schedule_interval=schedule_interval,
start_date=start_date,
)
task_clean_mo_names = BashOperator(task_id='clean_gexport_mo_names',
bash_command=r"""
sed -i -r "
s/_(BSC6900GSM|BSC6900UMTS|BSC6900GU|BSC6910GSM|BSC6910UMTS|BSC6910GU)//ig;
s/_(BTS3900|PICOBTS3900|BTS3911B|PICOBTS3911B|MICROBTS3900|MICROBTS3911B)//ig;
s/BSC(6910|6900)(UMTS|GSM)Function/FUNCTION/ig;
s/BSC(6910|6900)Equipment/EQUIPMENT/ig;
s/<class name=\"(.*)\"/<class name=\"\U\1\"/ig;
s/<class name=\"(.*)_MSCSERVER/<class name=\"\1/ig;
s/<class name=\"(.*)_ENODEB\"/<class name=\"\1\"/ig;
s/<class name=\"(.*)3900/<class name=\"\1/ig;
" /mediation/data/cm/huawei/raw/gexport/*.xml || true
""",
dag=dag)
parse_huawei_gexport_cm_files = BashOperator(
task_id='parse_huawei_gexport_cm_files',
bash_command=
'java -jar /mediation/bin/boda-huaweicmobjectparser.jar -i /mediation/data/cm/huawei/raw/gexport -o /mediation/data/cm/huawei/parsed/gexport -c /mediation/conf/cm/huawei_gexport_parser.cfg',
dag=dag)
import_nbi_csv = BashOperator(
task_id='import_huawei_gexport_parsed_csv',
bash_command=
'python /mediation/bin/load_cm_data_into_db.py huawei_gexport /mediation/data/cm/huawei/parsed/gexport',
dag=dag)
t_run_huawei_gexport_insert_queries = BashOperator(
task_id='run_huawei_gexport_insert_queries',
bash_command=
'python /mediation/bin/run_cm_load_insert_queries.py huawei_gexport',
dag=dag)
# Clear 4G CM data tables
def clear_huawei_gexport_cm_tables():
pass
t50 = PythonOperator(task_id='clear_huawei_gexport_cm_tables',
python_callable=clear_huawei_gexport_cm_tables,
dag=dag)
dag.set_dependency('clean_gexport_mo_names',
'parse_huawei_gexport_cm_files')
dag.set_dependency('parse_huawei_gexport_cm_files',
'clear_huawei_gexport_cm_tables')
dag.set_dependency('clear_huawei_gexport_cm_tables',
'import_huawei_gexport_parsed_csv')
dag.set_dependency('import_huawei_gexport_parsed_csv',
'run_huawei_gexport_insert_queries')
return dag
task_extract_zte_4g4g_nbrs = PythonOperator(
task_id='extract_zte_4g4g_nbrs',
python_callable=extract_zte_4g4g_nbrs,
dag=dag)
task_zte_cm_done = DummyOperator(task_id='zte_cm_done', dag=dag)
join_nokia_supported_task = DummyOperator(task_id='join_nokia_supported',
trigger_rule=TriggerRule.ONE_SUCCESS,
dag=dag)
# Build dependency graph
# dag.set_dependency('start_cm_load','is_ericsson_supported')
dag.set_dependency('start_cm_load', 'process_ericsson')
dag.set_dependency('ericsson_cm_done', 'join_ericsson_supported')
dag.set_dependency('join_ericsson_supported', 'end_cm_load')
dag.set_dependency('process_ericsson', 'parse_and_import_ericsson_bulkcm')
dag.set_dependency('parse_and_import_ericsson_bulkcm', 'backup_3g4g_raw_files')
dag.set_dependency('parse_and_import_ericsson_bulkcm', 'process_eri_rncs')
dag.set_dependency('parse_and_import_ericsson_bulkcm', 'process_eri_enodebs')
dag.set_dependency('process_eri_rncs', 'extract_ericsson_3g_sites')
dag.set_dependency('extract_ericsson_3g_sites', 'extract_ericsson_3g_cells')
dag.set_dependency('process_eri_enodebs', 'extract_ericsson_4g_cells')
dag.set_dependency('extract_ericsson_3g_cells', 'cell_extraction_done')
dag.set_dependency('backup_3g4g_raw_files', 'ericsson_cm_done')
