def __sub__(self, file_list):
subs_names = [item.name for item in file_list.list]
ep = EndPoint()
if self.update:
for item in self.list:
# upload file
try:
if item.mtime > self.updated[item.name]['mtime']:
print('>>>', item)
for file in file_list.list:
if file.name == item.name:
ep.remove({
'id': file.id,
})
print('delete', file.id)
subs_names.remove(item.name)
except AttributeError:
pass
# download file
try:
if item.version > self.updated[item.name]['version']:
subs_names.remove(item.name)
except AttributeError:
pass
return [item for item in self.list if item.name not in subs_names]
def __call__(self):
# super().__call__()
ep = EndPoint()
sess = ep.connect()
path, id_path = ep.get_full_path(sess['cwd'])
path = '/osdr/home/{username}/'.format(**sess) + path
id_path = '/{owner}/'.format(**sess) + id_path
print("Working directory path: {path}".format(path=path))
if self.verbosity > 0:
print("Working directory ID path: {path}".format(path=id_path))
def __call__(self):
# super().__call__()
ep = EndPoint()
session = ep.connect()
list_url = CONTENTS.format(session['cwd'])
record = ep.get_container_by_id(self.container)
if not record:
records = ep.get_containers(list_url)
record = ep.get_uniq_container(records, self.container)
folder = '.'
file_name = record['name']
if self.output:
if os.path.isdir(self.output):
folder = self.output
elif not os.path.dirname(self.output):
file_name = self.output
else:
file_name = os.path.basename(self.output)
folder = os.path.dirname(self.output)
assert os.path.isdir(folder), 'No such folder: {}'.format(folder)
path = os.path.join(folder, file_name)
if not self.overwrite:
assert not os.path.isfile(path), "File {path} exists. To overwrite use -f key.".format(path=path)
print(record)
ep.download(record, path)
def __call__(self):
# super().__call__()
ep = EndPoint()
ep.connect()
resp = ep.get(url=ME_URL)
try:
print("Login name: {loginName}".format(**resp.json()))
except Exception as e:
print(e)
messages = []
if self.verbosity > 0:
message = 'Logged in OSDR user "{displayName}"'
messages.append(message)
if self.verbosity > 1:
message = 'OSDR user id "{id}"'
messages.append(message)
for message in messages:
print(message.format(**resp.json()))
def __call__(self):
# super().__call__()
ep = EndPoint()
session = ep.connect()
mime_type = 'multipart/mixed'
list_url = CONTENTS.format(session['cwd'])
if self.container == '.':
self.container = session['cwd']
else:
record = ep.get_container_by_id(self.container)
if not record:
records = ep.get_containers(list_url)
record = ep.get_uniq_container(records, self.container)
assert record['type'] in ('User', 'Folder'), \
"Container '{name}' is not a folder".format(**record)
self.container = session['cwd'] = record['id']
cwd = session['cwd']
owner = session['owner']
files = {}
meta = {}
# prepare model data for uploading
if self.meta:
meta = MACHINE_LEARNING_MODEL
meta.update({
'UserId': cwd,
'ParentId': cwd,
})
model_meta = self.get_meta(filepath=self.meta)
meta['ModelInfo'] = json.dumps(model_meta)
# prepare file data for upload
if self.meta:
self.file = self.file[:1] # only one model allowed
mime_type = 'application/x-spss-sav'
total_names = self.name and len(self.name) or 0
for idx, file in enumerate(self.file):
if not os.path.isfile(file):
raise IOError('File %s not found' % file)
fh = open(file, 'rb')
if idx < total_names and self.name[idx]:
filename = self.name[idx]
else:
filename = os.path.basename(file)
if self.meta:
filename = model_meta.get('ModelName', filename)
files.update({'file%d' % idx: (filename, fh, mime_type)})
url = UPLOAD.format(id=owner)
meta.update({
'parentId': cwd,
})
resp = ep.post(url, meta, files=files)
def __call__(self):
# super().__call__()
ep = EndPoint()
session = ep.connect()
self.container = self.container[0]
list_url = CONTENTS.format(session['cwd'])
record = ep.get_container_by_id(self.container)
if not record:
records = ep.get_containers(list_url)
record = ep.get_uniq_container(records, self.container)
ep.remove(record)
def __call__(self):
ep = EndPoint()
session = ep.connect()
# get c sdf file
list_url = CONTENTS.format(session['cwd'])
record = ep.get_container_by_id(self.container)
if not record:
records = ep.get_containers(list_url)
record = ep.get_uniq_container(records, self.container)
assert record.get('status') == 'Processed', \
"File '{}' has not been processed yet".format(record['name'])
assert record['name'].lower().endswith('sdf'), \
"Unsupported file format"
model_name = self.__set_model_meta(record['name'])
# check folder name
records = ep.get_containers(list_url)
try:
ep.get_uniq_container(records, model_name)
raise RuntimeError(
"Model container '{}' already exists".format(model_name))
except AssertionError:
pass
placement = {
"parentId": session['cwd'],
"userId": session['owner'],
"sourceBlobId": record['blob']['id'],
"sourceBucket": record['blob']['bucket'],
}
self.model_meta.update(placement)
pprint.pprint(self.model_meta)
url = TRAIN
resp = ep.post(url, self.model_meta)
result = resp.json()
def __call__(self):
# super().__call__()
ep = EndPoint()
credentials = {
'username': self.username,
'password': self.password,
}
session = ep.authorize_remote(credentials)
# print(session)
messages = []
if self.verbosity >= 0:
message = 'Logged in successfully with username "{username}"'
messages.append(message)
if self.verbosity > 1:
message = 'User\'s OSDR root folder "{owner}"'
messages.append(message)
message = 'User\'s current working directory "{cwd}"'
messages.append(message)
if self.verbosity > 2:
message = 'Token: {token}'
messages.append(message)
for message in messages:
print(message.format(**session))
def __call__(self):
ep = EndPoint()
session = ep.connect()
list_url = CONTENTS.format(session['cwd'])
# Container
records = ep.get_containers(list_url)
try:
ep.get_uniq_container(records, self.folder)
raise RuntimeError(
"Model container '{}' already exists".format(self.folder))
except AssertionError:
pass
# Recordset
resp = ep.get(NODE.format(self.recordset))
recordset = resp.json()
assert recordset['subType'] == 'Records'
# Model
resp = ep.get(NODE.format(self.model))
model = resp.json()
assert model['subType'] == 'Model'
query = {
'UserId': session['owner'],
'ParentId': session['cwd'],
'FolderName': self.folder,
'DatasetBlobId': recordset['blob']['id'],
'DatasetBucket': recordset['blob']['bucket'],
'ModelBlobId': model['blob']['id'],
'ModelBucket': model['blob']['bucket'],
}
# print(query)
resp = ep.post(PREDICT, data=query)
result = resp.json()
assert result['modelFolderId'], \
"Error training model '{}'".format(model_name)
def __call__(self):
# super().__call__()
ep = EndPoint()
sess = ep.connect()
url = BROWSE_CONTENTS.format(cwd=sess['cwd'], page=1, size=100)
# Get container ID
url_params = dict(cwd=sess['cwd'], page=self.page, size=self.size)
if self.container:
record = ep.get_container_by_id(self.container)
if not record:
records = ep.get_containers(url)
record = ep.get_uniq_container(records, self.container)
if record:
url_params.update({
'cwd': record['id'],
})
url = BROWSE_CONTENTS.format(**url_params)
resp = ep.get(url)
records = resp.json()
if resp.headers.get('X-Pagination', None):
pages = json.loads(resp.headers['X-Pagination'])
page = self.pager_format.format(title='Page:', **pages)
print(page)
for key, ds in self.dataset.items():
self.dataset[key]['data'] = \
filter(lambda x: x['type'] == key, records)
for key in ('Folder File Records Record'.split()):
dataset = list(self.dataset[key]['data'])
if len(dataset):
print(colored.yellow(key))
for rec in sorted(dataset, key=lambda r: r.get('subType', '')):
fmt = self.dataset[key]['format']
if rec.get('totalRecords'):
fmt = self.dataset[key]['format_recs']
rep_rec = fmt.format(**rec)
if rec.get('status'):
status = rec['status']
status = status == 'Processed' \
and colored.green or colored.red
rep_rec = status(rep_rec)
print(rep_rec)
def __call__(self):
# super().__call__()
ep = EndPoint()
ep.remove_storage()
print('Successful logout')
def __call__(self):
# super().__call__()
ep = EndPoint()
session = ep.connect()
if self.container == '.':
ep.set_workdir(session['owner'])
return
if self.container == '..':
resp = ep.get(url=NODE.format(session['cwd']))
context = resp.json()
if context.get('parentId'):
ep.set_workdir(context.get('parentId'))
else:
print('You are at /osdr/home/{}'.format(session['username']))
return
list_url = CONTENTS.format(session['cwd'])
record = ep.get_container_by_id(self.container)
if not record:
records = ep.get_containers(list_url)
record = ep.get_uniq_container(records, self.container)
ep.set_workdir(record['id'])
def __call__(self):
# super().__call__()
ep = EndPoint()
session = ep.connect()
def __call__(self):
assert os.path.isdir(self.folder), \
"'{folder}' is not a folder".format(folder=self.folder)
ep = EndPoint()
session = ep.connect()
# Local files
lfiles = ListHelper(path=self.folder,
update=self.update_remote)
for file in os.listdir(self.folder):
path = os.path.join(self.folder, file)
rec = LocalFiles(name=os.path.basename(path),
mtime=os.path.getmtime(path))
if self._is_local_file(path):
lfiles.list.append(rec)
# remote folder id
list_url = CONTENTS.format(session['cwd'])
if self.container == '.':
self.container = session['cwd']
else:
record = ep.get_container_by_id(self.container)
if not record:
records = ep.get_containers(list_url)
record = ep.get_uniq_container(records, self.container)
assert record['type'] in ('User', 'Folder'), \
"Container '{name}' is not a folder".format(**record)
self.container = session['cwd'] = record['id']
list_url = CONTENTS.format(self.container)
# remote files
records = ep.get_containers(list_url)
records = list(records)
rfiles = ListHelper(self.folder, update=self.update_local)
rfiles.list = [RemoteFiles(name=rec['name'], id=rec['id'],
version=rec['version'],
length=rec['blob']['length'],
bucket=rec['blob']['bucket'],
bid=rec['blob']['id'])
for rec in filter(self._is_remote_file, records)]
# # files to download
print('\n\nDownloading...')
for file in rfiles - lfiles:
rec = {'type': 'File', 'name': file.name, 'length': file.length,
'blob': {'id': file.bid, 'bucket': file.bucket,
'length': file.length}}
path = os.path.join(self.folder, file.name)
try:
ep.download(rec, path=path)
lfiles.log(path=path, file=file)
except Exception as e:
print(e)
# file to upload
print('Uploading...')
for file in lfiles - rfiles:
path = os.path.join(self.folder, file.name)
try:
print('Uploading %s' % path)
ep.upload(session, path)
lfiles.log(path=path, file=file)
except Exception as e:
print(e)
lfiles.store_log()