def scan(**kwargs):
files = scan_folder(basepath)
# filter tension logs
files = dict(filter(lambda elem: "tension" in elem[0], files.items()))
for f in files:
logger.info(f)
kwargs['ti'].xcom_push(key='import_files', value=len(files))
def read_data(**kwargs):
files_list = []
projectid = None
importlog_id = None
# make list of files from folder
# data = kwargs['ti'].xcom_pull(key='import_files', task_ids='scan_folder')
files = scan_folder(basepath)
# filter ballasting logs
data = dict(filter(lambda elem: "instest" in elem[0], files.items()))
# for every file
for k, l in data.items():
files_list.append(l['file_path'])
log_path = l['file_path']
logger.info('files: {}'.format(l['file_path']))
# get Project name
projectname = l["project_name"]
# get Vessel name
vesselname = l["vessel_name"]
logger.info('project name: {}, vessel name: {}'.format(
projectname, vesselname))
# search Project_name + vessel_name in Project table
projectid = check_project_id(projectname, vesselname)
if projectid is not None:
# get id
logger.info('project id {} already exists'.format(projectid))
else:
logger.info('project doesn`t exist')
# add into Project table
save_project_name_toMySql(projectname, vesselname)
projectid = check_project_id(projectname, vesselname)
# get id
logger.info('project id {} already exists'.format(projectid))
# search importlog+project_name+vessel_name in importlog+Project tables
importlog_id = check_log_id(log_path, projectid)
if importlog_id is not None:
logger.info('importlog id {}'.format(importlog_id))
else:
logger.info('importlog file dosn`t exist')
# import file into insTestRes table
b = read_instest2(log_path)
instest_toMySql(b, projectid)
logger.info('IMPORTED RES')
# import file into insTestLimits table
lim_date = b['updated'][0]
lim = read_instestlimits2(log_path, lim_date)
instestlimits_toMySql(lim, projectid)
logger.info('IMPORTED LIMITS')
# add into importlog
save_log_toMySql(log_path, projectid)
logger.info('LOG is UPDATED')
def move(**kwargs):
files = scan_folder(basepath)
# filter ballasting logs
data = dict(filter(lambda elem: "instest" in elem[0], files.items()))
# task_instance = kwargs['ti']
# data = task_instance.xcom_pull(key='import_files', task_ids='scan_folder')
for s in data:
d = os.path.dirname(os.path.abspath(s))
d = str(s).replace('projects', 'archive')
shutil.move(s, d)
def scan(**kwargs):
files = scan_folder(basepath)
# filter instest logs
files = dict(filter(lambda elem: "instest" in elem[0], files.items()))
kwargs['ti'].xcom_push(key='import_files', value=len(files))