def upload_file(fname, cmd, rmcmd,snow_conn):
print('Starting %s' % fname)
compress_file(fname)
subprocess.call([cmd], shell=True)
to_file='%s.gz' % fname
s3fn = InOut()
s3fn.file_names=[]
s3fn.file_names.append(to_file)
Snowflake.bulk_copy( trans = snow_conn, file_names = s3fn, target=cli.tcfg, qname = 'copyStmt')
os.remove(to_file)
subprocess.call([rmcmd], shell=True)
cli, conn_pool = app_init
Dir = create_reader('Dir', app_init=app_init)
SQLite = create_writer('SQLite', app_init=app_init)
Email = create_actor('Email', app_init=app_init)
data_files = InOut()
lite_conn = InOut()
##
##
email_args = {'email_subject': 'File->SQLite'}
##
##
data_files.file_names = []
def run():
Dir.get_files(out=data_files)
if 1:
SQLite.begin_transaction(out=lite_conn)
SQLite.bulk_insert(trans=lite_conn,
file_names=data_files,
qname='insertStmt')
SQLite.commit_transaction(trans=lite_conn)
if 0:
Email.send_email(**email_args)
cli, conn_pool = app_init
Snowflake = create_writer('Snowflake', app_init=app_init)
S3 = create_writer('S3', app_init=app_init)
Email = create_actor('Email', app_init=app_init)
s3_file_names = InOut()
snow_conn = InOut()
##
##
email_args = {'email_subject': 'File->Snowflake'}
##
##
threads = {}
s3_file_names.file_names = []
def run():
Snowflake.begin_transaction(out=snow_conn)
assert isinstance(
cli.pa,
(str, unicode
)), 'Provide only file name you want to upload in [--proc_params]'
fname = cli.pa
s3_file_names.file_names
assert os.path.isfile(fname), 'Upload file "%s" does not exists.' % fname
if 1:
to_fn = '%s.gz' % fname
import subprocess
from collections import OrderedDict
from pprint import pprint as pp
from include.utils import create_reader, create_writer, create_actor, InOut
e = sys.exit
cli, conn_pool = app_init
Email = create_actor('Email', app_init=app_init)
ok_files = InOut()
out_files = InOut()
lite_conn = InOut()
ok_files.file_names = []
out_files.file_names = []
dump_file = InOut()
file_size_rows = 250000
email_args = {'email_subject': 'GTX->Vertica'}
insert_stats = InOut(source_cnt=-1, inserted_cnt=-1)
def run():
for _source, val in cli.cfg['dump'].items():
cli.set_source(_source)
_src_class = val.keys()[0]
DirReader = create_reader(_src_class, app_init=app_init)
"""
import sys
from pprint import pprint as pp
from include.utils import create_reader, create_writer, create_actor, InOut
e = sys.exit
cli, conn_pool = app_init
Email = create_actor('Email', app_init=app_init)
Dir = create_reader('Dir', app_init=app_init)
uploaded_files = InOut()
data_files = InOut()
data_files.file_names = []
uploaded_files.file_names = []
email_args = {'email_subject': 'DB->file'}
def run():
ext_files = []
for _source, val in cli.cfg['source'].items():
_dbname = val["sourceDb"]
DB = create_reader(_dbname, app_init=app_init)
FileWriter = create_writer('File', app_init=app_init)
data_files.file_names = []
uploaded_files.file_names = []
if 1: