def __call__(self):
# super().__call__()
ep = EndPoint()
session = ep.connect()
if self.container == '.':
ep.set_workdir(session['owner'])
return
if self.container == '..':
resp = ep.get(url=NODE.format(session['cwd']))
context = resp.json()
if context.get('parentId'):
ep.set_workdir(context.get('parentId'))
else:
print('You are at /osdr/home/{}'.format(session['username']))
return
list_url = CONTENTS.format(session['cwd'])
record = ep.get_container_by_id(self.container)
if not record:
records = ep.get_containers(list_url)
record = ep.get_uniq_container(records, self.container)
ep.set_workdir(record['id'])
def __call__(self):
# super().__call__()
ep = EndPoint()
session = ep.connect()
list_url = CONTENTS.format(session['cwd'])
record = ep.get_container_by_id(self.container)
if not record:
records = ep.get_containers(list_url)
record = ep.get_uniq_container(records, self.container)
folder = '.'
file_name = record['name']
if self.output:
if os.path.isdir(self.output):
folder = self.output
elif not os.path.dirname(self.output):
file_name = self.output
else:
file_name = os.path.basename(self.output)
folder = os.path.dirname(self.output)
assert os.path.isdir(folder), 'No such folder: {}'.format(folder)
path = os.path.join(folder, file_name)
if not self.overwrite:
assert not os.path.isfile(path), "File {path} exists. To overwrite use -f key.".format(path=path)
print(record)
ep.download(record, path)
def __call__(self):
# super().__call__()
ep = EndPoint()
session = ep.connect()
mime_type = 'multipart/mixed'
list_url = CONTENTS.format(session['cwd'])
if self.container == '.':
self.container = session['cwd']
else:
record = ep.get_container_by_id(self.container)
if not record:
records = ep.get_containers(list_url)
record = ep.get_uniq_container(records, self.container)
assert record['type'] in ('User', 'Folder'), \
"Container '{name}' is not a folder".format(**record)
self.container = session['cwd'] = record['id']
cwd = session['cwd']
owner = session['owner']
files = {}
meta = {}
# prepare model data for uploading
if self.meta:
meta = MACHINE_LEARNING_MODEL
meta.update({
'UserId': cwd,
'ParentId': cwd,
})
model_meta = self.get_meta(filepath=self.meta)
meta['ModelInfo'] = json.dumps(model_meta)
# prepare file data for upload
if self.meta:
self.file = self.file[:1] # only one model allowed
mime_type = 'application/x-spss-sav'
total_names = self.name and len(self.name) or 0
for idx, file in enumerate(self.file):
if not os.path.isfile(file):
raise IOError('File %s not found' % file)
fh = open(file, 'rb')
if idx < total_names and self.name[idx]:
filename = self.name[idx]
else:
filename = os.path.basename(file)
if self.meta:
filename = model_meta.get('ModelName', filename)
files.update({'file%d' % idx: (filename, fh, mime_type)})
url = UPLOAD.format(id=owner)
meta.update({
'parentId': cwd,
})
resp = ep.post(url, meta, files=files)
def __call__(self):
# super().__call__()
ep = EndPoint()
session = ep.connect()
self.container = self.container[0]
list_url = CONTENTS.format(session['cwd'])
record = ep.get_container_by_id(self.container)
if not record:
records = ep.get_containers(list_url)
record = ep.get_uniq_container(records, self.container)
ep.remove(record)
def __call__(self):
ep = EndPoint()
session = ep.connect()
# get c sdf file
list_url = CONTENTS.format(session['cwd'])
record = ep.get_container_by_id(self.container)
if not record:
records = ep.get_containers(list_url)
record = ep.get_uniq_container(records, self.container)
assert record.get('status') == 'Processed', \
"File '{}' has not been processed yet".format(record['name'])
assert record['name'].lower().endswith('sdf'), \
"Unsupported file format"
model_name = self.__set_model_meta(record['name'])
# check folder name
records = ep.get_containers(list_url)
try:
ep.get_uniq_container(records, model_name)
raise RuntimeError(
"Model container '{}' already exists".format(model_name))
except AssertionError:
pass
placement = {
"parentId": session['cwd'],
"userId": session['owner'],
"sourceBlobId": record['blob']['id'],
"sourceBucket": record['blob']['bucket'],
}
self.model_meta.update(placement)
pprint.pprint(self.model_meta)
url = TRAIN
resp = ep.post(url, self.model_meta)
result = resp.json()
def __call__(self):
ep = EndPoint()
session = ep.connect()
list_url = CONTENTS.format(session['cwd'])
# Container
records = ep.get_containers(list_url)
try:
ep.get_uniq_container(records, self.folder)
raise RuntimeError(
"Model container '{}' already exists".format(self.folder))
except AssertionError:
pass
# Recordset
resp = ep.get(NODE.format(self.recordset))
recordset = resp.json()
assert recordset['subType'] == 'Records'
# Model
resp = ep.get(NODE.format(self.model))
model = resp.json()
assert model['subType'] == 'Model'
query = {
'UserId': session['owner'],
'ParentId': session['cwd'],
'FolderName': self.folder,
'DatasetBlobId': recordset['blob']['id'],
'DatasetBucket': recordset['blob']['bucket'],
'ModelBlobId': model['blob']['id'],
'ModelBucket': model['blob']['bucket'],
}
# print(query)
resp = ep.post(PREDICT, data=query)
result = resp.json()
assert result['modelFolderId'], \
"Error training model '{}'".format(model_name)
def __call__(self):
assert os.path.isdir(self.folder), \
"'{folder}' is not a folder".format(folder=self.folder)
ep = EndPoint()
session = ep.connect()
# Local files
lfiles = ListHelper(path=self.folder,
update=self.update_remote)
for file in os.listdir(self.folder):
path = os.path.join(self.folder, file)
rec = LocalFiles(name=os.path.basename(path),
mtime=os.path.getmtime(path))
if self._is_local_file(path):
lfiles.list.append(rec)
# remote folder id
list_url = CONTENTS.format(session['cwd'])
if self.container == '.':
self.container = session['cwd']
else:
record = ep.get_container_by_id(self.container)
if not record:
records = ep.get_containers(list_url)
record = ep.get_uniq_container(records, self.container)
assert record['type'] in ('User', 'Folder'), \
"Container '{name}' is not a folder".format(**record)
self.container = session['cwd'] = record['id']
list_url = CONTENTS.format(self.container)
# remote files
records = ep.get_containers(list_url)
records = list(records)
rfiles = ListHelper(self.folder, update=self.update_local)
rfiles.list = [RemoteFiles(name=rec['name'], id=rec['id'],
version=rec['version'],
length=rec['blob']['length'],
bucket=rec['blob']['bucket'],
bid=rec['blob']['id'])
for rec in filter(self._is_remote_file, records)]
# # files to download
print('\n\nDownloading...')
for file in rfiles - lfiles:
rec = {'type': 'File', 'name': file.name, 'length': file.length,
'blob': {'id': file.bid, 'bucket': file.bucket,
'length': file.length}}
path = os.path.join(self.folder, file.name)
try:
ep.download(rec, path=path)
lfiles.log(path=path, file=file)
except Exception as e:
print(e)
# file to upload
print('Uploading...')
for file in lfiles - rfiles:
path = os.path.join(self.folder, file.name)
try:
print('Uploading %s' % path)
ep.upload(session, path)
lfiles.log(path=path, file=file)
except Exception as e:
print(e)
lfiles.store_log()