def cleanup_pipe():
d6tpipe.api.ConfigManager(profile=cfg['d6tpipe_profile']).init(
{'filerepo': 'tests/d6tpipe-files1/'}, reset=True)
api1 = d6tpipe.APIClient(profile=cfg['d6tpipe_profile'])
api1.login(cfg['d6tpipe_username'], cfg['d6tpipe_password'])
# d6tpipe.api.create_pipe_with_remote(api, {'name': cfg['d6tpipe_pipe1'], 'protocol': 'd6tfree'})
# settings = {"user":cfg['d6tpipe_username2'],"role":"read"}
# d6tpipe.create_or_update_permissions(api1, cfg['d6tpipe_pipe1'], settings)
pipe1 = d6tpipe.Pipe(api1, cfg['d6tpipe_pipe1'])
pipe1.delete_files_local(confirm=False, delete_all=True)
d6tpipe.api.ConfigManager(profile=cfg['d6tpipe_profile2']).init(
{'filerepo': 'tests/d6tpipe-files2/'}, reset=True)
api2 = d6tpipe.APIClient(profile=cfg['d6tpipe_profile2'])
api2.login(cfg['d6tpipe_username2'], cfg['d6tpipe_password2'])
# d6tpipe.api.create_pipe_with_remote(api2, {'name': cfg['d6tpipe_pipe2'], 'protocol': 'd6tfree'})
pipe12 = d6tpipe.Pipe(api2, cfg['d6tpipe_pipe1'])
pipe2 = d6tpipe.Pipe(api2, cfg['d6tpipe_pipe2'])
pipe12.delete_files_local(confirm=False, delete_all=True)
pipe2.delete_files_local(confirm=False, delete_all=True)
yield api1, api2
@fuckit
def delhelper():
pipe1.delete_files_local(confirm=False, delete_all=True)
pipe12.delete_files_local(confirm=False, delete_all=True)
pipe2.delete_files_local(confirm=False, delete_all=True)
delhelper()
def scan_local(pipe, profile):
pipe_name = pipe
try:
api = d6tpipe.api.APIClient(profile=profile)
pipe = d6tpipe.Pipe(api, pipe_name)
scan_result = pipe.scan_local()
print(json.dumps(scan_result, indent=4, sort_keys=True))
except Exception as e:
click.echo(e)
def push(pipe, profile, preview):
pipe_name = pipe
try:
api = d6tpipe.api.APIClient(profile=profile)
pipe = d6tpipe.Pipe(api, pipe_name)
if preview:
pipe.push_preview()
else:
pipe.push()
except Exception as e:
click.echo(e)
def init(default_pipe_name,
profile=None,
local_pipe=False,
local_api=False,
reset=False,
api=None,
set_dir=True,
api_args=None,
pipe_args=None):
"""
Initialize d6tpipe
Args:
default_pipe_name (str): name of pipe to store results. Override by setting Task.pipe attribute
profile (str): name of d6tpipe profile to get api if api not provided
local_pipe (bool): use `PipeLocal()`
local_api (bool): use `APILocal()`
reset (bool): reset api and pipe connection
api (obj): d6tpipe api object. if not provided will be loaded
set_dir (bool): if True, set d6tflow directory to default pipe directory
api_args (dir): arguments to pass to api
pipe_args (dir): arguments to pass to pipe
"""
if not d6tflow.settings.isinitpipe or reset:
d6tflow.cache.pipe_default_name = default_pipe_name
api_args = {} if api_args is None else api_args
pipe_args = {} if pipe_args is None else pipe_args
if local_pipe:
pipe_ = d6tpipe.PipeLocal(default_pipe_name, profile=profile)
else:
if api is None:
if local_api:
d6tflow.cache.api = d6tpipe.APILocal(profile=profile,
**api_args)
else:
d6tflow.cache.api = d6tpipe.APIClient(profile=profile,
**api_args)
else:
d6tflow.cache.api = api
pipe_ = d6tpipe.Pipe(d6tflow.cache.api, default_pipe_name,
**pipe_args)
d6tflow.cache.pipes[default_pipe_name] = pipe_
if set_dir:
d6tflow.settings.isinitpipe = False
d6tflow.set_dir(pipe_.dir)
d6tflow.settings.isinitpipe = True
import d6tpipe
#****************************
# vendor
#****************************
api = d6tpipe.APIClient(profile='d6tdev')
d6tpipe.upsert_pipe(api,{'name':'demo-vendor'})
pipe = d6tpipe.Pipe(api, 'demo-vendor')
print(pipe.settings['options']['remotepath'])
import pandas as pd
df = pd.DataFrame({'date':range(10),'data':range(10)})
df.to_csv(pipe.dirpath/'monthly-201901.csv')
df.to_csv(pipe.dirpath/'daily-201901.csv')
pipe.setmode('all')
pipe.push_preview()
pipe.push()
# create subscription levels
d6tpipe.upsert_pipe(api,{'name':'demo-vendor-daily','parent':'demo-vendor','options':{'include':'*daily*.csv'}})
d6tpipe.upsert_pipe(api,{'name':'demo-vendor-monthly','parent':'demo-vendor','options':{'include':'*monthly*.csv'}})
#****************************
# client permissions
#****************************
d6tpipe.api.ConfigManager(profile='demo').init() # use demo account
api2 = d6tpipe.api.APIClient(profile='demo')
def init_pipe(name=None, **kwargs):
_assertinit()
name = d6tflow.cache.pipe_default_name if name is None else name
pipe_ = d6tpipe.Pipe(d6tflow.cache.api, name, **kwargs)
d6tflow.cache.pipes[name] = pipe_