def _import(params, format, filename):
api.login(params)
if format == 'json':
def read_json():
with open(filename, 'rt') as f:
return json.load(f)
records_to_add = [api.prepare_record(params, parse_json(json)) for json in read_json()]
else:
def read_lines():
with open(filename, 'rt') as f:
return f.readlines()
records_to_add = [api.prepare_record(params, parse_line(line)) for line in read_lines()]
if (len(records_to_add) == 0):
print('No records to import')
return
request = api.make_request(params, 'record_update')
print('importing {0} records to Keeper'.format(len(records_to_add)))
request['add_records'] = records_to_add
response_json = api.communicate(params, request)
success = [info for info in response_json['add_records'] if info['status'] == 'success']
if len(success) > 0:
print("{0} records imported successfully".format(len(success)))
failures = [info for info in response_json['add_records'] if info['status'] != 'success']
if len(failures) > 0:
print("{0} records failed to import".format(len(failures)))
def delete_all(params):
api.sync_down(params)
if (len(params.record_cache) == 0):
print('No records to delete')
return
request = api.make_request(params, 'record_update')
print('removing {0} records from Keeper'.format(len(params.record_cache)))
request['delete_records'] = [key for key in params.record_cache.keys()]
response_json = api.communicate(params, request)
success = [info for info in response_json['delete_records'] if info['status'] == 'success']
if len(success) > 0:
print("{0} records deleted successfully".format(len(success)))
failures = [info for info in response_json['delete_records'] if info['status'] != 'success']
if len(failures) > 0:
print("{0} records failed to delete".format(len(failures)))
def upload_attachment(params, attachments):
'''
:param attachments:
:type attachments: [(str, ImportAttachment)]
'''
print('Uploading attachments:')
while len(attachments) > 0:
chunk = attachments[:90]
attachments = attachments[90:]
uploads = None
file_no = 0
file_size = 0
for _, att in chunk:
file_no += 1
file_size += att.size or 0
#TODO check storage subscription
rq = {'command': 'request_upload', 'file_count': file_no}
try:
rs = api.communicate(params, rq)
if rs['result'] == 'success':
uploads = rs['file_uploads']
except Exception as e:
logging.error(e)
return
uploaded = {}
for record_id, atta in chunk:
if not uploads:
break
try:
upload = uploads.pop()
buffer = io.BytesIO()
cipher = None
key = atta.key
if not key:
key = os.urandom(32)
iv = os.urandom(16)
cipher = AES.new(key, AES.MODE_CBC, iv)
buffer.write(iv)
with atta.open() as s:
finished = False
while not finished:
chunk = s.read(10240)
if len(chunk) > 0:
if cipher is not None:
if len(chunk) < 10240:
finished = True
chunk = api.pad_binary(chunk)
chunk = cipher.encrypt(chunk)
buffer.write(chunk)
else:
finished = True
size = buffer.tell() - 16
if size > 0:
buffer.seek(0, io.SEEK_SET)
files = {
upload['file_parameter']:
(atta.name, buffer, 'application/octet-stream')
}
print('{0} ... '.format(atta.name), end='', flush=True)
response = requests.post(upload['url'],
files=files,
data=upload['parameters'])
if response.status_code == upload['success_status_code']:
if record_id not in uploaded:
uploaded[record_id] = []
uploaded[record_id].append({
'key':
base64.urlsafe_b64encode(key).decode().rstrip('='),
'name':
atta.name,
'file_id':
upload['file_id'],
'size':
size
})
print('Done')
else:
print('Failed')
except Exception as e:
logging.warning(e)
if len(uploaded) > 0:
rq = {
'command': 'record_update',
'pt': 'Commander',
'device_id': 'Commander',
'client_time': api.current_milli_time(),
'update_records': []
}
for record_id in uploaded:
if record_id in params.record_cache:
rec = params.record_cache[record_id]
extra = json.loads(rec['extra_unencrypted'].decode(
'utf-8')) if 'extra' in rec else {}
files = extra.get('files')
if files is None:
files = []
extra['files'] = files
udata = rec['udata'] if 'udata' in rec else {}
file_ids = udata.get('file_ids')
if file_ids is None:
file_ids = []
udata['file_ids'] = file_ids
for atta in uploaded[record_id]:
file_ids.append(atta['file_id'])
files.append({
'id': atta['file_id'],
'name': atta['name'],
'size': atta['size'],
'key': atta['key']
})
ru = {
'record_uid':
record_id,
'version':
2,
'client_modified_time':
api.current_milli_time(),
'extra':
api.encrypt_aes(
json.dumps(extra).encode('utf-8'),
rec['record_key_unencrypted']),
'udata':
udata,
'revision':
rec['revision']
}
api.resolve_record_access_path(params, record_id, path=ru)
rq['update_records'].append(ru)
try:
rs = api.communicate(params, rq)
if rs['result'] == 'success':
api.sync_down(params)
except Exception as e:
logging.debug(e)
def prepare_folder_permission(params, folders):
shared_folder_lookup = {}
for shared_folder_uid in params.shared_folder_cache:
path = get_folder_path(params, shared_folder_uid)
if path:
shared_folder_lookup[path] = shared_folder_uid
email_pattern = re.compile(EMAIL_PATTERN)
# public keys
emails = {}
for fol in folders:
if fol.permissions:
for perm in fol.permissions:
if not perm.uid:
match = email_pattern.match(perm.name)
if match:
if perm.name not in emails:
if perm.name != params.user:
emails[perm.name.lower()] = None
if emails:
request = {'command': "public_keys", 'key_owners': list(emails.keys())}
try:
rs = api.communicate(params, request)
if 'public_keys' in rs:
for pk in rs['public_keys']:
if 'public_key' in pk:
emails[pk['key_owner']] = pk['public_key']
except Exception as e:
logging.debug(e)
folder_permissions = []
for fol in folders:
shared_folder_uid = shared_folder_lookup.get(fol.path)
if not shared_folder_uid:
continue
shared_folder = params.shared_folder_cache.get(shared_folder_uid)
if not shared_folder:
continue
shared_folder_key = shared_folder.get('shared_folder_key_unencrypted')
if not shared_folder_key:
continue
if fol.permissions:
add_users = []
add_teams = []
for perm in fol.permissions:
is_team = False
if perm.uid and params.team_cache:
is_team = perm.uid in params.team_cache
else:
match = email_pattern.match(perm.name)
if not match:
is_team = True
if is_team:
perm.uid = None
for team_uid in params.team_cache:
team = params.team_cache[team_uid]
if team['name'].lower() == perm.name.lower():
perm.uid = team_uid
break
if is_team:
team = params.team_cache.get(perm.uid)
if team:
found = False
if 'teams' in shared_folder:
for team in shared_folder['teams']:
if team['team_uid'] == team['team_uid']:
found = True
break
if not found:
add_teams.append({
'team_uid':
team['team_uid'],
'manage_users':
perm.manage_users,
'manage_records':
perm.manage_records,
'shared_folder_key':
api.encrypt_aes(shared_folder_key,
team['team_key_unencrypted']),
})
elif perm.name:
email = perm.name.lower()
found = False
if 'users' in shared_folder:
for user in shared_folder['users']:
if user['username'] == email:
found = True
break
if not found:
if email == params.user.lower():
add_users.append({
'username':
email,
'manage_users':
perm.manage_users,
'manage_records':
perm.manage_records,
'shared_folder_key':
api.encrypt_aes(shared_folder_key,
params.data_key)
})
elif email in emails:
public_key = emails[email]
if public_key:
try:
rsa_key = RSA.importKey(
base64.urlsafe_b64decode(public_key +
'=='))
add_users.append({
'username':
email,
'manage_users':
perm.manage_users,
'manage_records':
perm.manage_records,
'shared_folder_key':
api.encrypt_rsa(
shared_folder_key, rsa_key)
})
except Exception as e:
logging.debug(e)
if add_teams or add_users:
request = {
'command': 'shared_folder_update',
'operation': 'update',
'pt': 'Commander',
'shared_folder_uid': shared_folder_uid,
'force_update': True,
'add_teams': add_teams,
'add_users': add_users,
}
folder_permissions.append(request)
return folder_permissions
def prepare_shared_folder_add(params, folders):
folder_hash = {}
for f_uid in params.folder_cache:
fol = params.folder_cache[f_uid]
h = hashlib.md5()
hs = '{0}|{1}'.format((fol.name or '').lower(), fol.parent_uid or '')
h.update(hs.encode())
shared_folder_key = None
if fol.type in {
BaseFolderNode.SharedFolderType,
BaseFolderNode.SharedFolderFolderType
}:
sf_uid = fol.shared_folder_uid if fol.type == BaseFolderNode.SharedFolderFolderType else fol.uid
if sf_uid in params.shared_folder_cache:
shared_folder_key = params.shared_folder_cache[sf_uid][
'shared_folder_key_unencrypted']
folder_hash[h.hexdigest()] = f_uid, fol.type, shared_folder_key
# public keys
emails = {}
for fol in folders:
if fol.permissions:
for perm in fol.permissions:
if perm.name not in emails:
_, email = parseaddr(perm.name)
if email:
if email != params.user:
emails[email.lower()] = None
if emails:
request = {'command': "public_keys", 'key_owners': list(emails.keys())}
try:
rs = api.communicate(params, request)
if 'public_keys' in rs:
for pk in rs['public_keys']:
if 'public_key' in pk:
emails[pk['key_owner']] = pk['public_key']
except Exception as e:
logging.debug(e)
shared_folder_add = []
for fol in folders:
skip_folder = False
parent_uid = ''
parent_type = ''
parent_key = None
comps = list(path_components(fol.path))
for i in range(len(comps)):
comp = comps[i]
h = hashlib.md5()
hs = '{0}|{1}'.format(comp.lower(), parent_uid)
h.update(hs.encode())
digest = h.hexdigest()
is_last = False
if i == len(comps) - 1:
is_last = True
if digest not in folder_hash:
folder_uid = api.generate_record_uid()
request = {'command': 'folder_add', 'folder_uid': folder_uid}
folder_type = 'shared_folder' if is_last else 'user_folder'
request['folder_type'] = folder_type
encryption_key = params.data_key
folder_key = os.urandom(32)
request['key'] = api.encrypt_aes(folder_key, encryption_key)
if parent_uid:
request['parent_uid'] = parent_uid
if folder_type == 'shared_folder':
request['name'] = api.encrypt_aes(comp.encode('utf-8'),
folder_key)
data = {'name': comp}
request['data'] = api.encrypt_aes(
json.dumps(data).encode('utf-8'), folder_key)
shared_folder_add.append(request)
parent_uid = folder_uid
parent_type = folder_type
parent_key = folder_key
folder_hash[
digest] = folder_uid, folder_type, folder_key if folder_type == 'shared_folder' else None
else:
parent_uid, parent_type, parent_key = folder_hash[digest]
if is_last:
skip_folder = parent_type != 'shared_folder'
else:
skip_folder = parent_type != 'user_folder'
if skip_folder:
break
if not skip_folder and parent_type == 'shared_folder':
request = {
'command': 'shared_folder_update',
'operation': 'update',
'pt': 'Commander',
'shared_folder_uid': parent_uid,
'force_update': True,
'default_manage_users': fol.manage_users,
'default_manage_records': fol.manage_records,
'default_can_edit': fol.can_edit,
'default_can_share': fol.can_share
}
if fol.permissions:
for perm in fol.permissions:
is_team = False
if perm.uid and params.team_cache:
is_team = perm.uid in params.team_cache
else:
_, email = parseaddr(perm.name)
if not email:
is_team = True
if is_team:
perm.uid = None
for team in params.team_cache:
if team['name'].lower() == perm.name.lower():
perm.uid = team['team_uid']
break
if is_team:
if perm.uid and perm.uid in params.team_cache:
if 'add_teams' not in request:
request['add_teams'] = []
team = params.team_cache[perm.uid]
request['add_teams'].append({
'team_uid':
perm.uid,
'manage_users':
perm.manage_users,
'manage_records':
perm.manage_records,
'shared_folder_key':
api.encrypt_aes(parent_key, team['team_key'])
})
else:
if 'add_users' not in request:
request['add_users'] = []
email = perm.name.lower()
if email == params.user.lower():
request['add_users'].append({
'username':
email,
'manage_users':
perm.manage_users,
'manage_records':
perm.manage_records,
'shared_folder_key':
api.encrypt_aes(parent_key, params.data_key)
})
elif email in emails:
public_key = emails[email]
if public_key:
try:
rsa_key = RSA.importKey(
base64.urlsafe_b64decode(public_key +
'=='))
request['add_users'].append({
'username':
email,
'manage_users':
perm.manage_users,
'manage_records':
perm.manage_records,
'shared_folder_key':
api.encrypt_rsa(parent_key, rsa_key)
})
except:
pass
shared_folder_add.append(request)
return shared_folder_add