def action_process(d: DataDir, config: Configuration, logger: Logger,
userid: int):
if not os.path.exists(d.user_config_path(userid)):
raise ValueError(f'config file for user {userid} not found')
user_data = DataDir(config)
user_data.add_config(d.user_config_path(userid))
acc: UserAccounts = d.UserAccounts(userid)
# try to convert xls to xlsx
for datafile in d.UserXLSDataFiles(userid):
try:
import pyexcel as p
xlsx_file = datafile + 'x'
logger.info(f'Trying to convert {datafile} to {xlsx_file}')
if os.path.exists(xlsx_file):
logger.info(f'IGNORED (Already exists) {xlsx_file}')
else:
p.save_book_as(file_name=datafile, dest_file_name=xlsx_file)
logger.info(f'Converted')
except ImportError as e:
logger.error('Error during xls to xlsx conversion',
'Error',
exc_info=e)
datafiles: List[str] = d.UserDataFiles(userid)
if len(datafiles) == 0:
logger.error('No xlsx files')
import_tag: str = f'IMP-{datetime.date.today().strftime("%Y-%m-%d")}'
for datafile in d.UserDataFiles(userid):
#datafile_parts:str = os.path.splitext(os.path.basename(datafile))
outputfn = os.path.join(user_data.user_folder_output(userid),
os.path.basename(datafile))
stmt_data = XlsxDriver(filepath=datafile,
output_file_path=outputfn,
first_row=get_config_int(
user_data.config, 'datafile.first_row'))
stmt_data.read()
from execution_context import ExecutionContext
ec = ExecutionContext(stmt_data, acc, list_files('rules', r'.*\.yaml'),
d.UserTaxNumber(userid),
d.user_folder_output(userid))
(total_rows, rows_processed) = ec.execute(import_tag)
stmt_data.close()
logger.info(
f'DONE. Processed {total_rows} rows, prepared {rows_processed} statements with tag {import_tag}'
)
def action_push(d: DataDir, config: Configuration, logger: Logger,
userid: int):
if not os.path.exists(d.user_config_path(userid)):
raise ValueError(f'config file for user {userid} not found')
user_data = DataDir(config)
user_data.add_config(d.user_config_path(userid))
output_folder = d.user_folder_output(userid)
if not os.path.exists(output_folder):
return
# raise ValueError(f'folder does not exist {output_folder}')
processed_files = 0
for f in sorted(os.listdir(output_folder)):
m = re.match(
f'(?P<index>[0-9]+)-(?P<operation>\w+)-(?P<operation_id>\w+).json',
f)
if not m: continue
index = m.group('index')
operation = m.group('operation')
operation_id = m.group('operation_id')
print(f'{index} : processing {operation} / {operation_id}')
with open(os.path.join(output_folder, f),
encoding='utf8') as json_file:
data = ujson.load(json_file)
user_data.ProcessOperation(operation, userid, data)
processed_files = processed_files + 1
print(f'DONE. Processed {processed_files} statements')
else:
p.save_book_as(file_name=datafile,
dest_file_name=xlsx_file)
logger.info(f'Converted')
except ImportError as e:
logger.error('Error during xls to xlsx conversion',
'Error',
exc_info=e)
datafiles: List[str] = d.UserDataFiles(userid)
if len(datafiles) == 0:
logger.error('No xlsx files')
import_tag: str = f'IMP-{datetime.date.today().strftime("%Y-%m-%d")}'
for datafile in d.UserDataFiles(userid):
#datafile_parts:str = os.path.splitext(os.path.basename(datafile))
outputfn = os.path.join(user_data.user_folder_output(userid),
os.path.basename(datafile))
stmt_data = XlsxDriver(filepath=datafile,
output_file_path=outputfn,
first_row=get_config_int(
user_data.config, 'datafile.first_row'))
stmt_data.read()
from execution_context import ExecutionContext
ec = ExecutionContext(stmt_data, d.UserAccounts(userid),
list_files('rules', r'.*\.yaml'),
d.UserTaxNumber(userid),
d.user_folder_output(userid))
(total_rows, rows_processed) = ec.execute(import_tag)