class Box:
def __init__(self):
#p=Proxy()
#p.URL='https://bcpxy.nycnet:8080'
#p.AUTH={'user': '******', 'password':'******'}
#self.session=Session(proxy_config=p)
self.session=Session()
self.sdk = JWTAuth.from_settings_file('/code/mocj_porter/config/box_config.json')
self.client=Client(self.sdk,self.session)
def create_folder(self,name, parent_id):
try:
self.client.folder(folder_id=parent_id).create_subfolder(name)
except Exception as e:
return {'status': 400, 'message': e}
else:
return {'status':200, 'message': 'Success'}
def post_file(self,s3bucket,file_config,box_folder_id):
file=storage.S3File(s3bucket,file_config)
file_bytes=BytesIO(file.get_file_bytes())
print('####hEEEYYYY#####')
#self.client.folder(box_folder_id).upload_stream(file_bytes, file_config['filename'])
self.client.folder(box_folder_id).upload('/code/mocj_porter/test.csv','test')
return {'status':200, 'message': 'Success'}
def download_from_box(self,
files_to_download,
box_folder=None,
use_dl_links=False):
"""
Downloads a list of files from a specified Box folder
Optionally, you can simply grab the download link to send to another user
"""
oauth2 = OAuth2(self.clientId,
self.clientSecret,
access_token=self.accessToken)
client = Client(oauth2)
my = client.user(user_id='me').get()
print('connected to Box as ' + my.login)
if box_folder is None:
target_folder = client.folder(self.box_folder_id)
else:
target_folder = client.folder(box_folder)
if use_dl_links is True:
link_array = []
target_folder_info = target_folder.get()
items_in_Box_folder = target_folder.get_items(limit=None, offset=0)
for file_name in files_to_download:
# Check the Box folder to see if the files exist
for item in items_in_Box_folder:
# If the file does already exist, use update_contents
if item.name == file_name and use_dl_links is False:
with open(self.temp_file_dir + file_name, 'wb') as f:
f.write(item.content())
print(file_name + ' downloaded')
elif item.name == file_name and use_dl_links is True:
box_link = item.get_shared_link_download_url()
link_array.append(box_link)
if use_dl_links is True:
return link_array
def _unsafe_get_box_object_from_oid(self, client: Client, oid: str, object_type: OType, strict: bool) \
-> Optional[BoxItem]:
assert isinstance(client, Client)
assert object_type in (FILE, DIRECTORY)
box_object = None
try:
with self._api():
if object_type == FILE:
box_object = client.file(file_id=oid)
if object_type == DIRECTORY:
box_object = client.folder(folder_id=oid)
if box_object:
box_object = self._unsafe_box_object_populate(client, box_object)
return box_object
except CloudFileNotFoundError:
pass
except (CloudFileExistsError, PermissionError):
raise
except Exception as e:
log.exception(e)
raise
# try again with the other type
log.debug("Trying again")
if object_type == FILE:
box_object = client.folder(folder_id=oid)
if object_type == DIRECTORY:
box_object = client.file(file_id=oid)
box_object = self._unsafe_box_object_populate(client, box_object) # should raise FNF if the object doesn't exists
if strict: # if we are here, then the object exists and retval does not comply with "strict"
raise CloudFileExistsError()
return box_object
class BoxJobSaver(JobSaver):
def __init__(self):
oauth = OAuth2(
client_id=config.BOX_CLIENT_ID,
client_secret=config.BOX_CLIENT_SECRET,
access_token=config.BOX_ACCESS_TOKEN,
)
self.client = Client(oauth)
self.local_job_saver = LocalJobSaver()
self.root_folder = self.client.folder(folder_id='0')
def save_job_offers(self, job_offers) -> str:
file_name = self.local_job_saver.save_job_offers(job_offers)
folder = self.create_folder_if_not_exist(REPORT_PATH)
uploaded_file = folder.upload(file_name)
return uploaded_file.get_shared_link()
def create_folder_if_not_exist(self, folder_path):
folder_exist, folder = self.folder_exist(folder_path)
if folder_exist:
return folder
else:
return self.root_folder.create_subfolder(folder_path)
def folder_exist(self, folder_path):
folders = self.root_folder.get_items()
for folder in folders:
if folder.type.capitalize(
) == 'Folder' and folder.name == folder_path:
return True, self.client.folder(folder_id=folder.id)
return False, None
def upload_to_box(self, files_to_upload, box_folder=None):
"""
Uploads a list of files to Box based on the folder id specified when FileMover is initialized
"""
oauth2 = OAuth2(self.clientId,
self.clientSecret,
access_token=self.accessToken)
client = Client(oauth2)
my = client.user(user_id='me').get()
print('connected to Box as ' + my.login)
if box_folder is None:
target_folder = client.folder(self.box_folder_id)
else:
target_folder = client.folder(box_folder)
target_folder_info = target_folder.get()
items_in_Box_folder = target_folder.get_items(limit=None, offset=0)
for file_name in files_to_upload:
# Check the Box folder to see if the files exist
for item in items_in_Box_folder:
# If the file does already exist, use update_contents
if item.name == file_name:
box_file = item.update_contents(self.temp_file_dir +
file_name)
print(item.name + ' updated')
break
# If the file did not exist, use upload
if file_name not in set(files_to_upload):
box_file = target_folder.upload(self.temp_file_dir + file_name,
file_name)
print(file_name + ' uploaded')
os.remove(self.temp_file_dir + file_name)
def upload():
auth = JWTAuth(client_id='t7o9oknjyfmavpz2vq8qyeh2tq3bgwkc',
client_secret='tYjGJzz1iMiPA2JBW6csMyAFEXPrUj2c',
enterprise_id='511092066',
jwt_key_id='u4zebqgi',
rsa_private_key_file_sys_path='private.pem')
access_token = auth.authenticate_instance()
client = Client(auth)
user_to_impersonate = client.user(user_id='13315944066')
client = client.as_user(user_to_impersonate)
user = client.user().get()
print('The current user ID is {0}'.format(user.id))
subfolder = client.folder('0').create_subfolder(str(Supportid))
print('Created subfolder with ID {0}'.format(subfolder.id))
cmdfile = client.folder(subfolder.id).upload('specs-cmd.txt')
print(
'File "{0}" uploaded to Circuit Technical Spec Storage with file ID {1}'
.format(cmdfile.name, cmdfile.id))
psfile = client.folder(subfolder.id).upload('specs-ps.txt')
print(
'File "{0}" uploaded to Circuit Technical Spec Storage with file ID {1}'
.format(psfile.name, psfile.id))
pass
class TransferData():
# Define client ID, client secret, and developer token.
def __init__(self):
self.CLIENT_ID = None
self.CLIENT_SECRET = None
self.ACCESS_TOKEN = None
try:
with open('box.cfg', 'r') as box_cfg:
self.CLIENT_ID = box_cfg.readline().rstrip('\n')
self.CLIENT_SECRET = box_cfg.readline().rstrip('\n')
self.ACCESS_TOKEN = box_cfg.readline().rstrip('\n')
# Create OAuth2 object.
self.oauth2 = OAuth2(self.CLIENT_ID,
self.CLIENT_SECRET,
access_token=self.ACCESS_TOKEN)
# Create the authenticated client
self.client = Client(self.oauth2)
except:
print("box.cfg not found for boxsdk")
print(
"please make an box.cfg file with CLIENT_ID,CLIENT_SECRET, and ACCESS_TOKEN on 3 lines."
)
traceback.print_exc()
exit()
def upload_file(self, file_from):
"""upload a file to Box
"""
try:
self.client.folder('0').upload(file_from,
file_from,
preflight_check=True)
except:
pass
os.remove(file_from)
def download_file(self, file_from):
"""download a file from Box
"""
file_list = self.client.folder(folder_id='0').get_items(limit=100,
offset=0)
for file1 in file_list:
if (file1['name'] == file_from):
#print("downloaded from box")
with open(file_from, 'wb') as open_file:
self.client.file(file1.id).download_to(open_file)
open_file.close()
def delete_file(self, file_from):
"""delete file from box
"""
file_list = self.client.folder(folder_id='0').get_items(limit=100,
offset=0)
for file1 in file_list:
if (file1['name'] == file_from):
self.client.file(file1.id).delete()
def get_items_in_Box_folder(self, box_folder=None):
oauth2 = OAuth2(self.clientId,
self.clientSecret,
access_token=self.accessToken)
client = Client(oauth2)
my = client.user(user_id='me').get()
if box_folder is None:
target_folder = client.folder(self.box_folder_id)
else:
target_folder = client.folder(box_folder)
items_in_Box_folder = target_folder.get_items(limit=None, offset=0)
return items_in_Box_folder
def create_folder(args, user_integration):
bxc = BOX_Credentials.objects.get(user_integration_id=user_integration)
ACCESS_TOKEN = bxc.BOX_ACCESS_TOKEN
REFRESH_TOKEN = bxc.BOX_REFRESH_TOKEN
#print(ACCESS_TOKEN)
REFRESH_TOKEN = bxc.BOX_REFRESH_TOKEN
#print(REFRESH_TOKEN + " refreshtoken")
# Log the token we're using & starting call
logging.info('Using Refresh Token: %s' % REFRESH_TOKEN)
# Get new access & refresh tokens
getTokens = requests.post(oauth2URL,
data={
'grant_type': 'refresh_token',
'refresh_token': REFRESH_TOKEN,
'client_id': clientId,
'client_secret': clientSecret
})
# If the above gives a 4XX or 5XX error
# getTokens.raise_for_status()
# Get the JSON from the above
newTokens = getTokens.json()
# Get the new access token, valid for 60 minutes
accessToken = newTokens['access_token']
refreshToken = newTokens['refresh_token']
#print("New accessToken " + accessToken)
#print("New refreshToken " + refreshToken)
bxc.BOX_REFRESH_TOKEN = refreshToken
bxc.BOX_ACCESS_TOKEN = accessToken
bxc.save()
CLIENT_ID = settings.CLIENT_ID
CLIENT_SECRET = settings.CLIENT_SECRET
oauth2 = OAuth2(CLIENT_ID, CLIENT_SECRET, access_token=ACCESS_TOKEN)
name = args['Folder-Name']
client = Client(oauth2, LoggingNetwork())
#items = client.folder(folder_id='0').get_items(limit=1000, offset=0)
#print("List of all files and folders")
#print("created "+name)
m = MessageClass()
client.folder(folder_id='0').create_subfolder(name)
field = AttachmentFieldsClass()
field.title = "Successfully created the folder " + name
attachment = MessageAttachmentsClass()
attachment.attach_field(field)
m.attach(attachment)
return m
def get(self, request, *args, **kwargs):
from io import BytesIO
from django.http import HttpResponse
boxuser = BoxUser.objects.order_by('-id')[0]
oauth = RedisManagedOAuth2(
client_id='5dn98104cyf535v4581cbb1wxnag6e5y',
client_secret='8z6ysMEnsrickMWBwpnysxYJ9SvqaNlY',
unique_id=boxuser.unique_id
)
client = Client(oauth)
folder_items = (
client.folder(folder_id='0').get_items(limit=100, offset=0))
file_items = [f for f in folder_items if f.type == "file"]
first_item_id = file_items[0].id
file_name = client.file(file_id=first_item_id).get()['name']
# RESPONSE
response = HttpResponse(content_type='application/octet-stream')
response['Content-Disposition'] = (
'attachment; filename="%s"' % file_name)
content = BytesIO(client.file(file_id=first_item_id).content())
file_content = content.getvalue()
content.close()
response.write(file_content)
return response
def accessUploadFolder(year=2020):
# Define client ID, client secret, and developer token.path = os.path.join(*[os.path.dirname(os.path.abspath(__file__)),"instance"])
# Read app info from text file
config = ConfigObject(
os.path.join(*[
os.path.dirname(os.path.abspath(__file__)), "instance",
'Boxapp.cfg'
]))
CLIENT_ID = config['client_id']
CLIENT_FOLDER = config['client_folder' + str(year)]
ACCESS_TOKEN = config['access_token']
# Create OAuth2 object.
auth = OAuth2(client_id=CLIENT_ID,
client_secret='',
access_token=ACCESS_TOKEN)
# Create the authenticated client
client = Client(auth)
# make sure we connected
try:
my = client.user(user_id='me').get()
print(my.name) # developer name tied to the token
except:
sys.exit("ERROR: Invalid access token; try re-generating an "
"access token from the app console on the web.")
tfolder = client.folder(CLIENT_FOLDER) # 2020 scada data folder
return tfolder
def worker(self):
while True:
folder_id = self.folders_queue.get()
if folder_id is None:
break
logging.info(f'getting items for {folder_id}')
client = Client(self.auth)
# store folder items in list as generator can be interated once
current_folder_items = list(
client.folder(folder_id=folder_id).get_items(
fields=self.fields))
# add the subfolders to the queue
subfolders = filter(lambda f: f.type == 'folder',
current_folder_items)
for s in subfolders:
self.folders_queue.put(s.id)
# run the callback on each folder item
for item in current_folder_items:
self.func(item)
# pause to keep API calls in check
time.sleep(self.wait)
self.folders_queue.task_done()
def get_fund_materials():
#authentication
#link to get Box developer token: https://app.box.com/developers/console/app/524950/configuration
dev_token = str(raw_input("Developer token: "))
oauth = OAuth2(
client_id='l91tu18y9v1t9yth4w3xub87jbyln18k',
client_secret='BYSJR7wr4Tl7Socbw3l87FYK01OOE91r',
access_token=dev_token,
)
#go to fund folder
client = Client(oauth)
root_folder = client.folder(folder_id='0')
USIT_folder = root_folder.get_items(limit=100, offset=0)[0]
fund_folder = USIT_folder.get_items(limit=100, offset=0)[1]
#adds files in each fund folder to the database
for item in fund_folder.get_items(limit=100, offset=0):
folder_name = item.get()['name']
#print folder_name
for file in item.get_items(limit=100, offset=0):
try:
download_url = file.get_shared_link_download_url()
file_name = file.get()['name']
code = download_url.split("/static/")[1].split(".")[0]
file_url = "https://app.box.com/embed/s/" + code
print(file_url)
file = FundFile(name = file_name, filePath=file_url, owner='Box', fund= folder_name)
db.session.add(file)
db.session.commit()
print("I added " + file_name + " to the database!")
except:
print("Cannot add file!")
def get_ag_materials():
dev_token = str(raw_input("Developer token: "))
oauth = OAuth2(
client_id='l91tu18y9v1t9yth4w3xub87jbyln18k',
client_secret='BYSJR7wr4Tl7Socbw3l87FYK01OOE91r',
access_token=dev_token
)
client = Client(oauth)
root_folder = client.folder(folder_id='0')
USIT_folder = root_folder.get_items(limit=100, offset=0)[0]
# for item in USIT_folder.get_items(limit=100, offset=0):
# print item.get()['name']
ag_folder = USIT_folder.get_items(limit=100, offset=0)[0]
for item in ag_folder.get_items(limit=100, offset=0):
try:
file_name = item.get()['name']
download_url = item.get_shared_link_download_url()
code = download_url.split("/static/")[1].split(".")[0]
file_url = "https://app.box.com/embed/s/" + code
# print(file_url)
file = AnalystFile(name = file_name, filePath = file_url, owner = 'Box')
db.session.add(file)
db.session.commit()
print("I added " + file_name + " to the database!")
except:
print("Cannot add file!")
def _folder_data(self, folder_id):
# Split out from set_folder for ease of testing, due to
# outgoing requests. Should only be called by set_folder
try:
Provider(self.external_account).refresh_oauth_key(force=True)
except InvalidGrantError:
raise exceptions.InvalidAuthError()
try:
oauth = OAuth2(client_id=settings.BOX_KEY,
client_secret=settings.BOX_SECRET,
access_token=ensure_str(
self.external_account.oauth_key))
client = Client(oauth)
folder_data = client.folder(self.folder_id).get()
except BoxAPIException:
raise exceptions.InvalidFolderError()
folder_name = folder_data['name'].replace('All Files',
'') or '/ (Full Box)'
folder_path = '/'.join([
x['name']
for x in folder_data['path_collection']['entries'] if x['name']
] + [folder_data['name']]).replace('All Files', '') or '/'
return folder_name, folder_path
class BoxAdaptor:
def __init__(self, path_to_config):
print("box_config", path_to_config)
if os.path.isfile(path_to_config) == False:
raise ValueError(
"configPath must be a path to the JSON config file for your Box JWT app"
)
auth = JWTAuth.from_settings_file(path_to_config)
logger.info("Authenticating BoxAdaptor...")
auth.authenticate_instance()
self.client = Client(auth)
def create_upload_folder(self, folder_name=None):
if not folder_name:
folder_name = str(uuid.uuid4())
folder = self.client.folder(0).create_subfolder(folder_name)
return folder
def upload_file(self, box_folder, local_path, dest_file_name):
logger.info(f"uploading: {dest_file_name}")
logger.info(f"to: {box_folder}")
logger.info(f"from: {local_path}")
box_file = box_folder.upload(local_path, dest_file_name)
return box_file
def set_retention_policy(self, folder):
policy = self.client.create_retention_policy(
policy_name="auto_delete",
disposition_action="permanently_delete",
retention_length=1,
)
policy.assign(folder)
def user_detail(user_id):
print '### Sending detail view ###'
client = Client(g.auth, network_layer=customLogger)
user = client.user(user_id=user_id).get()
# As an admin, we can act on behalf of other users by creating new auth and client objects.
# We should also be caching this token. For the purposes of this quickstart
# we only cache access for one user (the admin).
print "AUTHENTICATING USER: "******" (" + user.name + ")"
user_auth = JWTAuth(client_id=app.config['CLIENT_ID'],
client_secret=app.config['CLIENT_SECRET'],
enterprise_id=app.config['EID'],
jwt_key_id=app.config['KEY_ID'],
rsa_private_key_file_sys_path=os.path.join(os.path.dirname(__file__),'rsakey.pem'))
user_auth.authenticate_app_user(user) # <--- Authenticate as the user
user_client = Client(user_auth)
# Make API calls as the user by using the user_client object
files = user_client.folder(folder_id='0').get_items(limit=100)
# Build the preview link into any files sent to the client
for f in files:
if f._item_type=="file":
f.preview_url = f.get(fields=['expiring_embed_link']).expiring_embed_link['url']
token = user_auth.access_token
return render_template("user_detail.html",
user=user,
files_list=files,
token=token)
class SendToBox(object):
def __init__(self, config_filename='config.json'):
self.client = None
self.config_filename = config_filename
self.authenticate()
def authenticate(self):
'''
JWT authentication
'''
auth = JWTAuth.from_settings_file(self.config_filename)
access_token = auth.authenticate_instance()
self.client = Client(auth)
def upload_file(self, filename):
'''
using a try will attempt to upload file to box. If successful, will delete the file
from local system. If unsuccessful, will log error message to log file and
NOT delete the file
'''
try:
box_file = self.client.folder('0').upload(filename,
preflight_check=True)
os.remove(filename)
except BoxAPIException as err:
self.log_failure(filename, err)
def log_failure(self, file_name, err):
log_file_name = "failed_uploads.log"
logging.basicConfig(filename=log_file_name, level=logging.ERROR)
l1 = logging.getLogger(file_name)
l1.error(f"File failed to upload:\n {err}\n")
return
def push_to_box(folderid, file_path):
CLIENT_ID = Config.query.filter_by(key='boxclientid').value(Config.value)
CLIENT_SECRET = Config.query.filter_by(key='boxclientsecret').value(
Config.value)
oauth = OAuth2(client_id=CLIENT_ID,
client_secret=CLIENT_SECRET,
store_tokens=save_tokens)
tokens = read_tokens()
at = tokens['access_token']
rt = tokens['refresh_token']
oauth._access_token = at
oauth._refresh_token = rt
client = Client(oauth)
archive = client.folder(folder_id=folderid)
try:
upload_file = archive.upload(file_path)
try:
file_url = upload_file.get_shared_link_download_url(access='open')
return file_url
except Exception as e:
flash('Error getting shared link from Box: '.format(Exception),
category='danger')
return False
except Exception as e:
flash('Error uploading to Box: '.format(Exception), category='danger')
return False
def folder_detail(folder_id):
client = Client(g.auth, network_layer=customLogger)
folder = client.folder(folder_id=folder_id).get()
files = folder.get_items(limit=100)
return render_template("folder_detail.html",
folder=folder,
files_list=files)
def get_folders(self, **kwargs):
folder_id = kwargs.get('folder_id')
if folder_id is None:
return [{
'id': '0',
'path': '/',
'addon': 'box',
'kind': 'folder',
'name': '/ (Full Box)',
'urls': {
# 'folders': node.api_url_for('box_folder_list', folderId=0),
'folders':
api_v2_url('nodes/{}/addons/box/folders/'.format(
self.owner._id),
params={'id': '0'})
}
}]
try:
Provider(self.external_account).refresh_oauth_key()
oauth = OAuth2(client_id=settings.BOX_KEY,
client_secret=settings.BOX_SECRET,
access_token=ensure_str(
self.external_account.oauth_key))
client = Client(oauth)
except BoxAPIException:
raise HTTPError(http_status.HTTP_403_FORBIDDEN)
try:
metadata = client.folder(folder_id).get()
except BoxAPIException:
raise HTTPError(http_status.HTTP_404_NOT_FOUND)
except MaxRetryError:
raise HTTPError(http_status.HTTP_400_BAD_REQUEST)
folder_path = '/'.join(
[x['name'] for x in metadata['path_collection']['entries']] +
[metadata['name']])
return [{
'addon':
'box',
'kind':
'folder',
'id':
item['id'],
'name':
item['name'],
'path':
os.path.join(folder_path, item['name']).replace('All Files', ''),
'urls': {
'folders':
api_v2_url('nodes/{}/addons/box/folders/'.format(
self.owner._id),
params={'id': item['id']})
}
} for item in metadata['item_collection']['entries']
if item['type'] == 'folder']
def upload_to_box(report_path, deployment):
box_auth_string = environment.get_env(environment.BOX_AUTH_JSON)
box_auth_json = json.loads(box_auth_string.replace("'", ""))
box_sdk = JWTAuth.from_settings_dictionary(box_auth_json)
client = BoxClient(box_sdk)
file_name = create_file_name(deployment)
# Folder id 0 is the root folder
box_file = client.folder('0').upload(report_path, file_name)
return box_file.get_shared_link(access='open')
def list_all(args, user_integration):
#print("LIST")
bxc = BOX_Credentials.objects.get(user_integration_id=user_integration)
ACCESS_TOKEN = bxc.BOX_ACCESS_TOKEN
#print(ACCESS_TOKEN)
REFRESH_TOKEN = bxc.BOX_REFRESH_TOKEN
#print(REFRESH_TOKEN+" refreshtoken")
# Log the token we're using & starting call
logging.info('Using Refresh Token: %s' % REFRESH_TOKEN)
# Get new access & refresh tokens
#print("logged")
getTokens = requests.post(oauth2URL,
data={
'grant_type': 'refresh_token',
'refresh_token': REFRESH_TOKEN,
'client_id': clientId,
'client_secret': clientSecret
})
#print("GOT TOKENS")
# If the above gives a 4XX or 5XX error
#getTokens.raise_for_status()
# Get the JSON from the above
newTokens = getTokens.json()
#print("GOT NEW TOKEN")
# Get the new access token, valid for 60 minutes
accessToken = newTokens['access_token']
refreshToken = newTokens['refresh_token']
# print("New accessToken " + accessToken)
# print("New refreshToken " + refreshToken)
bxc.BOX_REFRESH_TOKEN = refreshToken
bxc.BOX_ACCESS_TOKEN = accessToken
bxc.save()
# Get the new access token, valid for 60 minutes
CLIENT_ID = settings.CLIENT_ID
CLIENT_SECRET = settings.CLIENT_SECRET
oauth2 = OAuth2(CLIENT_ID, CLIENT_SECRET, access_token=accessToken)
#print("listing...")
client = Client(oauth2, LoggingNetwork())
items = client.folder(folder_id='0').get_items(limit=1000, offset=0)
#print("List of all files and folders\n")
attachment = MessageAttachmentsClass()
m = MessageClass()
x = ''
for item in items:
field = AttachmentFieldsClass()
field.title = item['name']
field.value = item['id']
#print("Name: "+item['name']+" ID: "+item['id'])
x = x + "Name: " + item['name'] + " ID: " + item['id'] + "\n"
attachment.attach_field(field)
m.attach(attachment)
return m
def box_callback():
code = request.args.get('code')
access_token, refresh_token = oauth.authenticate(code)
client = Client(oauth)
file_path = '/Users/davidbliu/desktop/wd/filtering/comp/IMG_0860_comp.png'
file_name = id_generator()+'.png'
folder_id = '7173575673'
box_file = client.folder(folder_id).upload(file_path, file_name)
print box_file
return 'uploaded box file'
def box_callback():
code = request.args.get('code')
access_token, refresh_token = oauth.authenticate(code)
client = Client(oauth)
file_path = '/Users/davidbliu/desktop/wd/filtering/comp/IMG_0860_comp.png'
file_name = id_generator() + '.png'
folder_id = '7173575673'
box_file = client.folder(folder_id).upload(file_path, file_name)
print box_file
return 'uploaded box file'
def get_folders(self, **kwargs):
folder_id = kwargs.get('folder_id')
if folder_id is None:
return [{
'id': '0',
'path': '/',
'addon': 'box',
'kind': 'folder',
'name': '/ (Full Box)',
'urls': {
# 'folders': node.api_url_for('box_folder_list', folderId=0),
'folders': api_v2_url('nodes/{}/addons/box/folders/'.format(self.owner._id),
params={'id': '0'}
)
}
}]
try:
Provider(self.external_account).refresh_oauth_key()
oauth = OAuth2(client_id=settings.BOX_KEY, client_secret=settings.BOX_SECRET, access_token=self.external_account.oauth_key)
client = Client(oauth)
except BoxAPIException:
raise HTTPError(http.FORBIDDEN)
try:
metadata = client.folder(folder_id).get()
except BoxAPIException:
raise HTTPError(http.NOT_FOUND)
except MaxRetryError:
raise HTTPError(http.BAD_REQUEST)
folder_path = '/'.join(
[
x['name']
for x in metadata['path_collection']['entries']
] + [metadata['name']]
)
return [
{
'addon': 'box',
'kind': 'folder',
'id': item['id'],
'name': item['name'],
'path': os.path.join(folder_path, item['name']).replace('All Files', ''),
'urls': {
'folders': api_v2_url('nodes/{}/addons/box/folders/'.format(self.owner._id),
params={'id': item['id']}
)
}
}
for item in metadata['item_collection']['entries']
if item['type'] == 'folder'
]
def main():
oauth = OAuth2(
client_id='<CLIENT_ID>',
client_secret='<CLIENT_SECRET>',
access_token='<ACCESS_TOKEN>'# manual input
)
client = Client(oauth=oauth)
box_contentlists = client.folder('0').get_items(100, 0, None)
print(box_contentlists)
return (box_contentlists)
class BoxClient:
def __init__(self):
oauth = OAuth2(client_id=BOX_CLIENT_ID,
client_secret=BOX_CLIENT_SECRET,
access_token=BOX_ACCESS_TOKEN)
self._client = Client(oauth)
def get_file(self, remote_folder_id, file_name):
items = self._client.folder(remote_folder_id).get_items(1000)
for item in items:
if item.name == file_name:
return item
return None
def upload_file(self, local_file_path, remote_folder_id, file_name):
return self._client.folder(remote_folder_id).upload(
local_file_path, file_name)
def download_file(self, remote_folder_id, file_name, local_path):
remote_file = self.get_file(remote_folder_id, file_name)
local_file = open(local_path, 'w')
remote_file.download_to(local_file)
local_file.close()
def get_folder(self, remote_folder_id):
return self._client.folder(remote_folder_id)
# Note that download_folder will recursively download all subdirectories.
def download_folder(self, remote_folder_id, local_folder_path):
# In case it doesn't exist, create local_folder_path.
if not os.path.exists(local_folder_path):
os.makedirs(local_folder_path)
items = self._client.folder(remote_folder_id).get_items(1000)
for item in items:
if item.type == 'file':
self.download_file(remote_folder_id, item.name,
'/'.join([local_folder_path, item.name]))
elif item.type == 'folder':
self.download_folder(item.id,
'/'.join([local_folder_path, item.name]))
def create_Box_folder(self, box_folder, folder_array):
oauth2 = OAuth2(self.clientId,
self.clientSecret,
access_token=self.accessToken)
client = Client(oauth2)
my = client.user(user_id='me').get()
target_folder = client.folder(box_folder)
if isinstance(folder_array, str):
target_folder.create_subfolder(folder_array)
else:
for folder in folder_array:
target_folder.create_subfolder(folder)
def main():
"""
Dictates what is to be done for the current process.
Current steps:
-Connect to s3 and download specified files to tempdir
-Generate Refresh token and store in logs
-Retrieve token from logs and authenticate
-Process files by renaming them
-Upload files to Box
-Remove the locally stored files
"""
downloadFilesFromS3(files_to_download_array=files_to_download,
s3Bucket_name=bucket_name)
accessToken = str(getToken())
oauth2 = OAuth2(clientId, clientSecret, access_token=accessToken)
client = Client(oauth2)
my = client.user(user_id='me').get()
print('connected to Box as ' + my.login)
target_folder = client.folder(box_folder_id)
# target_folder = client.folder('0')
target_folder_info = target_folder.get()
items_in_Box_folder = target_folder.get_items(limit=None, offset=0)
# Grab all of the files in the temp dir
files = listdir(temp_file_dir)
files_to_download_set = set(files_to_download)
# Start to upload the files to Box
print('uploading files to folder - ' + target_folder_info.name)
upload_array = []
for filename in files:
# Check to see if the file is one that exists in files_to_download
if filename in files_to_download_set:
filename_path = temp_file_dir + filename
# Drop the 000's from the end of the filename
new_filename = filename[:-3]
rename(filename_path, temp_file_dir + new_filename)
# Check the Box folder to see if the files exist
for item in items_in_Box_folder:
# If the file does already exist, use update_contents
if item.name == new_filename:
box_file = item.update_contents(filename_path[:-3])
print(item.name + ' updated')
upload_array.append(new_filename)
break
# If the file did not exist, use upload
if new_filename not in set(upload_array):
box_file = target_folder.upload(filename_path[:-3],
new_filename)
print(new_filename + ' uploaded')
os.remove(temp_file_dir + new_filename)
print('all files uploaded')
def main():
box_info = loadBoxInfo()
# Get oauth object
try:
oauth = box_login(box_info['CLIENT_ID'], box_info['CLIENT_SECRET'],
box_info['EMAIL'], box_info['PASSWORD'])
except ValueError:
print(
"\n********** Value Error! Double check your credentials in the config.ini file. **********\n"
)
exit()
# Create Box client
client = Client(oauth)
# Get Box root folder
root = client.folder(folder_id='0')
root_folder_items = root.get_items(limit=1000, offset=0)
# Get latest folder
latest_folder = root_folder_items[len(root_folder_items) - 1]
latest_folder_items = latest_folder.get_items(limit=100, offset=0)
# Create a folder in the local root directory with the name of the latest folder in Box
#local_folder = create_local_folder(str(latest_folder))
local_folder = create_local_folder('')
""" DOWNLOADS ALL IMAGES FROM BOX"""
for i in range(len(root_folder_items)):
_latest_folder_items = root_folder_items[i].get_items(limit=100,
offset=0)
for j in range(len(_latest_folder_items)):
image_path = os.path.join(local_folder,
str(_latest_folder_items[j]) + ".jpg")
if os.path.exists(image_path):
print("Exists: ", image_path)
continue
byte_array = _latest_folder_items[j].content()
f = open(image_path, "wb")
f.write(byte_array)
f.close()
print(image_path, " completed.")
print("Box scrape complete.")
""" DOWNLOAD LATEST FOLDER ONLY
def upload_files(file_paths, dir_name):
""" TODO """
config = JWTAuth.from_settings_file('static/secrets/config.json')
try:
client = Client(config)
content = Folder(client.folder('0'))
user = client.user().get()
print(content)
print(content.get_items())
print('The current user ID is {0}'.format(user.id))
except (BoxOAuthException, BoxException) as e:
# logging.warn()
raise e
def create_folder(folder_name):
"""
Creates a folder in the root folder given its name.
:param folder_name: Folder name to create.
:return: Folder identifier if the creation was successful, None otherwise.
"""
box_client = Client(JWTAuth.from_settings_file(BOX_CONFIG_FILE_PATH))
for i in range(0, BOX_RETRIES):
try:
sub_folder = box_client.folder(BOX_FOLDER_ROOT_ID).create_subfolder(folder_name)
return sub_folder.id
except Exception as e:
time.sleep(BOX_RTM)
if i == BOX_RETRIES - 1:
print(f'Error calling Box API creating the folder [{folder_name}] into folder root: {e}')
return None
def fill_folder_path_map(
box_client: boxsdk.Client,
folder_id: str,
seed_root_folder_id: str,
box_user: Optional[boxsdk.object.user.User] = None,
map_key_root: Optional[str] = None,
folder_path_map: Optional[dict] = None,
):
if not folder_path_map:
folder_path_map = dict()
if box_user:
folder = box_client.as_user(box_user).folder(folder_id).get()
else:
folder = box_client.folder(folder_id).get()
log.debug(f"got Box folder {folder}")
if folder_id != seed_root_folder_id:
folder_name = folder.response_object["name"]
if map_key_root:
map_key_root = f"{map_key_root}|{folder_name}"
else:
map_key_root = folder_name
folder_path_map[map_key_root] = folder
log.debug(
f"added folder map key {map_key_root} with value {folder_id} to Box folder path id map"
)
for folder_item in folder.get_items():
if folder_item.response_object["type"] == "folder":
folder_path_map = {
**folder_path_map,
**fill_folder_path_map(
box_client,
folder_item.response_object["id"],
seed_root_folder_id,
box_user,
map_key_root,
folder_path_map,
),
}
return folder_path_map
def upload_file(folder_id, file_path):
"""
Uploads a file (that must not exist in Box folder) into a folder given its path.
:param folder_id: Folder identifier.
:param file_path: File path.
:return: File identifier if the upload was successful, None otherwise.
"""
box_client = Client(JWTAuth.from_settings_file(BOX_CONFIG_FILE_PATH))
for i in range(0, BOX_RETRIES):
try:
file_name = file_path.split('/')[-1]
return box_client.folder(folder_id).upload(file_path, file_name).id
except Exception as e:
time.sleep(BOX_RTM)
if i == BOX_RETRIES - 1:
print(f'Error calling Box API uploading the file [{file_path}] to folder with id [{folder_id}]: {e}')
return None
def _folder_data(self, folder_id):
# Split out from set_folder for ease of testing, due to
# outgoing requests. Should only be called by set_folder
try:
Provider(self.external_account).refresh_oauth_key(force=True)
except InvalidGrantError:
raise exceptions.InvalidAuthError()
try:
oauth = OAuth2(client_id=settings.BOX_KEY, client_secret=settings.BOX_SECRET, access_token=self.external_account.oauth_key)
client = Client(oauth)
folder_data = client.folder(self.folder_id).get()
except BoxAPIException:
raise exceptions.InvalidFolderError()
folder_name = folder_data['name'].replace('All Files', '') or '/ (Full Box)'
folder_path = '/'.join(
[x['name'] for x in folder_data['path_collection']['entries'] if x['name']] +
[folder_data['name']]
).replace('All Files', '') or '/'
return folder_name, folder_path
def get_context_data(self, **kwargs):
context = super(HomeView, self).get_context_data(**kwargs)
if BoxUser.objects.count():
boxuser = BoxUser.objects.order_by('-id')[0]
oauth = RedisManagedOAuth2(
client_id='5dn98104cyf535v4581cbb1wxnag6e5y',
client_secret='8z6ysMEnsrickMWBwpnysxYJ9SvqaNlY',
unique_id=boxuser.unique_id
)
client = Client(oauth)
me = client.user(user_id='me').get()
folder_items = (
client.folder(folder_id='0').get_items(limit=100, offset=0))
context['boxuser'] = me
context['folder_items'] = folder_items
####
otherbox = BoxUser.objects.exclude(id=boxuser.id)
context['others'] = []
for other in otherbox:
otherauth = RedisManagedOAuth2(
client_id='5dn98104cyf535v4581cbb1wxnag6e5y',
client_secret='8z6ysMEnsrickMWBwpnysxYJ9SvqaNlY',
unique_id=other.unique_id
)
otherclient = Client(otherauth)
otherme = otherclient.user(user_id='me').get()
context['others'].append({
'boxuser': otherme,
'folder_items': (
otherclient
.folder(folder_id='0')
.get_items(limit=100, offset=0))
})
return context
def sendfiles2box(remotepath, filenames, remotefilenames=None,
overwrite=False):
"""Send a file(s) to Box.
Args:
remotepath (str): remote directory to send file(s) to
filenames (str or list): local files to send to box
remotefilenames (str or list, optional): same size as filenames and
optional as to if they should have different names or not
overwrite (bool): should this overwrite existing files, default `False`
Returns:
list of ids of the uploaded content
"""
if isinstance(filenames, string_types):
filenames = [filenames, ]
if isinstance(remotefilenames, string_types):
remotefilenames = [remotefilenames, ]
if remotefilenames is None:
remotefilenames = [os.path.basename(f) for f in filenames]
iemprops = get_properties()
oauth = OAuth2(
client_id=iemprops['boxclient.client_id'],
client_secret=iemprops['boxclient.client_secret'],
access_token=iemprops['boxclient.access_token'],
refresh_token=iemprops['boxclient.refresh_token'],
store_tokens=_store_tokens
)
client = Client(oauth)
folder_id = 0
for token in remotepath.split("/"):
if token.strip() == '':
continue
offset = 0
found = False
while not found:
LOG.debug("folder(%s).get_items(offset=%s)", folder_id, offset)
items = client.folder(
folder_id=folder_id).get_items(limit=100, offset=offset)
for item in items:
if (item.type == 'folder' and
item.name.lower() == token.lower()):
folder_id = item.id
found = True
break
if len(items) != 100:
break
offset += 100
if not found:
LOG.debug("Creating folder %s inside of %s", token, folder_id)
item = client.folder(folder_id=folder_id).create_subfolder(token)
folder_id = item.id
LOG.debug("Now we upload to folder_id: %s", folder_id)
res = []
for localfn, remotefn in zip(filenames, remotefilenames):
LOG.debug("uploading %s", localfn)
try:
item = client.folder(folder_id=folder_id).upload(localfn, remotefn)
res.append(item.id)
except Exception as exp:
if overwrite and hasattr(exp, 'context_info'):
_fileid = exp.context_info['conflicts']['id']
LOG.info("overwriting %s fid: %s", remotefn, _fileid)
try:
item = client.file(_fileid).update_contents(localfn)
res.append(_fileid)
continue
except Exception as exp2:
LOG.debug(
"Upload_Contents of %s resulted in exception: %s",
localfn, exp2
)
continue
LOG.debug(
"Upload of %s resulted in exception: %s", localfn, exp
)
res.append(None)
return res
webbrowser.open(auth_url)
auth_code_is_available.wait()
local_server.stop()
assert auth_code['state'] == csrf_token
access_token, refresh_token = oauth.authenticate(auth_code['auth_code'])
print('access_token: ' + access_token)
print('refresh_token: ' + refresh_token)
return oauth
if __name__ == '__main__':
client = Client(authenticate())
bam = client.folder(folder_id="6547362613").get_items(limit = 100000)
cram = client.folder(folder_id="6560143373").get_items(limit = 100000)
[x.create_shared_link(access = "open") for x in bam + cram]
files = {x["name"]:x.get_shared_link_download_url() for x in bam + cram if x.get().size > 0}
dict_out = {}
for f, link in files.items():
isotype = f.split(".")[0]
if isotype not in dict_out.keys():
dict_out[isotype] = {}
dict_out[isotype]["bam"] = ""
dict_out[isotype]["bam.bai"] = ""
dict_out[isotype]["cram"] = ""
dict_out[isotype]["cram.crai"] = ""
dict_out[isotype]["tsv"] = ""
dict_out[isotype][".".join(f.split(".")[1:])] = link
with open("static/content/data/urls.tsv", 'w') as f:
class BoxAPI(StorageAPI):
def __init__(self, credential_dir, credential_filename=CREDENTIAL_FILENAME):
super(BoxAPI, self).__init__(credential_dir)
self.auth_file = join(credential_dir, credential_filename)
self.oauth = OAuth2(client_id=CLIENT_ID, client_secret=CLIENT_SECRET, store_tokens=self.write_access_token)
try:
self.get_tokens_from_file()
self.authorize()
except IOError:
pass
def get_auth_url(self):
auth_url, csrf_token = self.oauth.get_authorization_url(REDIRECT_URI)
return auth_url
def build(self):
self.client = Client(self.oauth)
self.create_folder("demerio")
def create_folder(self, folder_name):
search_results = self.client.search(folder_name, limit=100, offset=0, ancestor_folders=[self.client.folder(folder_id='0')])
folder_filter = [result for result in search_results if result._item_type == "folder"]
if len(folder_filter) == 0:
demerio_folder = self.client.folder(folder_id='0').create_subfolder('demerio')
else:
assert len(folder_filter) == 1
demerio_folder = folder_filter[0].get(fields=["name"])
self.root_folder_id = demerio_folder.id
def get_tokens_from_file(self):
with open(self.auth_file, "r") as f:
access_token = f.readline().rstrip()
refresh_token = f.readline().rstrip()
return access_token, refresh_token
def write_access_token(self, access_token, refresh_token):
with open(self.auth_file, 'w') as f:
f.write(access_token + "\n")
f.write(refresh_token + "\n")
def authorize(self):
if os.path.exists(self.auth_file):
access_token, refresh_token = self.get_tokens_from_file()
self.oauth._access_token = access_token
self.oauth._refresh_token = refresh_token
else:
httpd = ClientRedirectServer(("localhost", 8888), ClientRedirectHandler)
webbrowser.open(self.get_auth_url())
httpd.handle_request()
self.oauth.authenticate(httpd.query_params['code'])
self.build()
def is_connected(self):
## TODO: There must be a better way to check connection, with self.oauth ??
try:
self.client.user(user_id='me').get()
except:
return False
return True
def download_file(self, file_id, path_to_download):
with open(path_to_download, "wb") as f:
f.write(self.client.file(file_id=file_id).content())
def upload_new_file(self, local_file_path):
new_file = self.client.folder(folder_id=self.root_folder_id).upload(local_file_path, file_name = generate_random_string())
return new_file.get()['id']
def delete_file(self, file_id):
self.client.file(file_id=file_id).delete()
def update_file(self, local_file_path, file_id):
self.client.file(file_id=file_id).update_contents(local_file_path)
class LoggingNetwork(DefaultNetwork):
def request(self, method, url, access_token, **kwargs):
""" Base class override. Pretty-prints outgoing requests and incoming responses. """
print ('\x1b[36m{} {} {}\x1b[0m'.format(method, url, pformat(kwargs)))
response = super(LoggingNetwork, self).request(
method, url, access_token, **kwargs
)
if response.ok:
print ('\x1b[32m{}\x1b[0m'.format(response.content))
else:
print ('\x1b[31m{}\n{}\n{}\x1b[0m'.format(
response.status_code,
response.headers,
pformat(response.content),
))
return response
oauth = OAuth2(CLIENT_ID, CLIENT_SECRET, access_token=DEVELOPER_TOKEN)
client = Client(oauth, LoggingNetwork())
root_folder = client.folder(folder_id='0').get()
items = client.folder(folder_id='0').get_items(limit=100, offset=0)
shared_link = client.folder(folder_id='3800889110').get_shared_link()
class Box(object):
_CLIENT_ID = Configuration.CLIENT_ID
_CLIENT_SECRET = Configuration.CLIENT_SECRET
_ENTERPRISE_ID = Configuration.ENTERPRISE_ID
_PASSPHRASE = Configuration.PASSPHRASE
def __init__(self):
self._db_engine = sqlalchemy.create_engine('sqlite+pysqlite:///photobooth.db')
self._session_maker = sessionmaker(bind=self._db_engine, autoflush=True)
self._session = self._session_maker()
DeclarativeBase.metadata.create_all(self._db_engine)
self._auth = JWTAuth(
client_id=self._CLIENT_ID,
client_secret=self._CLIENT_SECRET,
enterprise_id=self._ENTERPRISE_ID,
rsa_private_key_file_sys_path='private_key.pem',
rsa_private_key_passphrase=self._PASSPHRASE,
)
self._client = Client(self._auth)
try:
user_id = self._session.query(PhotoBoothInfo).filter_by(key='user_id').one().value
from boxsdk.object.user import User
self._upload_user = User(None, user_id)
except NoResultFound:
self._upload_user = self._client.create_user('Photobooth Uploader')
self._session.add(PhotoBoothInfo(key='user_id', value=self._upload_user.object_id))
self._session.commit()
self._uploader_auth = JWTAuth(
client_id=self._CLIENT_ID,
client_secret=self._CLIENT_SECRET,
enterprise_id=self._ENTERPRISE_ID,
rsa_private_key_file_sys_path='private_key.pem',
rsa_private_key_passphrase=self._PASSPHRASE,
)
self._uploader_auth.authenticate_app_user(self._upload_user)
self._uploader = Client(self._uploader_auth)
try:
folder_id = self._session.query(PhotoBoothInfo).filter_by(key='folder_id').one().value
self._folder = self._uploader.folder(folder_id)
except NoResultFound:
self._folder = self._uploader.folder('0').create_subfolder('Photobooth Images')
self._session.add(PhotoBoothInfo(key='folder_id', value=self._folder.object_id))
self._session.commit()
def upload_photo(self, name, message, photo_sys_path):
print 'uploading photo ', photo_sys_path, ' to box'
photo = self._folder.upload(photo_sys_path)
photo.metadata().create({
'name': name,
'message': message,
})
def download_photo(self, file_id, photo_sys_path):
print 'downloading photo ', photo_sys_path, ' from box'
with open(photo_sys_path, 'wb') as file_handle:
self._client.file(file_id).download_to(file_handle)
def list_files(self):
return self._folder.get_items(1000)
class Session(object):
"""Represents a ongoing / running session with Box API"""
IEM_PROPERTIES_ACCESS_TOKEN = 'boxclient.access_token'
IEM_PROPERTIES_REFRESH_TOKEN = 'boxclient.refresh_token'
IEM_PROPERTIES_CLIENT_ID = 'boxclient.client_id'
IEM_PROPERTIES_CLIENT_SECRET = 'boxclient.client_secret'
def __init__(self, client_id=None, client_secret=None,
access_token=None, refresh_token=None, store_tokens=None):
"""constructor
Args:
client_id (str): The application box client_id
client_secret (str): The application box client_secret
access_token (str): The Oauth2 access_token
refresh_token (str): The Oauth2 refresh_token
store_tokens (function): The Oauth2 callback on new tokens
"""
st = self.iem_token_callback if store_tokens is None else store_tokens
if client_id is None:
self.dbbootstrap(st)
else:
self.client_id = client_id
self.client_secret = client_secret
oauth = OAuth2(client_id=self.client_id,
client_secret=self.client_secret,
access_token=access_token,
refresh_token=refresh_token, store_tokens=st)
self.client = Client(oauth)
def dbbootstrap(self, store_tokens):
"""Get configuration from IEM Database"""
pgconn = psycopg2.connect(database='mesosite', host='iemdb')
cursor = pgconn.cursor()
cursor.execute("""SELECT propvalue from properties where
propname = %s""", (self.IEM_PROPERTIES_CLIENT_ID,))
self.client_id = cursor.fetchone()[0]
cursor.execute("""SELECT propvalue from properties where
propname = %s""", (self.IEM_PROPERTIES_CLIENT_SECRET,))
self.client_secret = cursor.fetchone()[0]
cursor.execute("""SELECT propvalue from properties where
propname = %s""", (self.IEM_PROPERTIES_ACCESS_TOKEN,))
access_token = cursor.fetchone()[0]
cursor.execute("""SELECT propvalue from properties where
propname = %s""", (self.IEM_PROPERTIES_REFRESH_TOKEN,))
refresh_token = cursor.fetchone()[0]
oauth = OAuth2(client_id=self.client_id,
client_secret=self.client_secret,
access_token=access_token,
refresh_token=refresh_token, store_tokens=store_tokens)
self.client = Client(oauth)
def iem_token_callback(self, access_token, refresh_token):
oauth = OAuth2(client_id=self.client_id,
client_secret=self.client_secret,
access_token=access_token,
refresh_token=refresh_token,
store_tokens=self.iem_token_callback)
self.client = Client(oauth)
pgconn = psycopg2.connect(database='mesosite', host='iemdb')
cursor = pgconn.cursor()
for propname, propvalue in zip([self.IEM_PROPERTIES_ACCESS_TOKEN,
self.IEM_PROPERTIES_REFRESH_TOKEN],
[access_token, refresh_token]):
cursor.execute("""
UPDATE properties SET propvalue = %s WHERE propname = %s
""", (propvalue, propname))
cursor.close()
pgconn.commit()
def get_folder(self, remote_folder):
"""Get or Create a remote folder on Box
Args:
remote_folder (str): the full remote path of the folder
"""
# print("get_folder(%s)" % (repr(remote_folder),))
dirs = remote_folder.split("/")
root = self.client.folder(folder_id=0)
for dirname in dirs:
if dirname == '':
continue
# BUG folders over 1000 items :/
found = False
for item in root.get_items(1000):
if item.name == dirname:
root = self.client.folder(item.object_id)
found = True
break
if not found:
root = root.create_subfolder(dirname)
return root
def rmirror(self, local_folder, remote_folder):
"""Recursively send local_folder to remote_folder"""
for root, _, filenames in os.walk(local_folder):
boxpath = os.path.join(remote_folder, root.lstrip(local_folder))
localfns = ["%s/%s" % (root, f) for f in filenames]
self.uploads(localfns, boxpath, filenames)
def upload(self, localfn, remote_folder, remotefn=None):
"""Upload a single file to remote path"""
remotefn = localfn if remotefn is None else remotefn
self.uploads([localfn, ], remote_folder, [remotefn, ])
def uploads(self, localfns, remote_folder, remotefns=[]):
"""Upload multiple files to remote path"""
root = self.get_folder(remote_folder)
currentitems = {}
for item in root.get_items(1000):
currentitems[item.name] = item
remotefns = localfns if len(remotefns) == 0 else remotefns
for localfn, remotefn in tqdm(zip(localfns, remotefns),
desc=remote_folder,
disable=(not sys.stdout.isatty())):
if remotefn in currentitems:
continue
root.upload(localfn, remotefn if remotefn is not None else localfn)
def process_event(self, event, operation):
"""
Wrapper to process the given event on the operation.
:param event:
:param operation:
:return:
"""
if operation == 'delete':
crate_logger.debug('Doing a delete on {}'.format(event.pathname))
folders_to_traverse = self.folders_to_traverse(event.path)
crate_logger.debug(folders_to_traverse)
client = Client(self.oauth)
box_folder = client.folder(folder_id='0').get()
cur_box_folder = box_folder
# if we're modifying in root box dir, then we've already found the folder
is_base = BOX_DIR in (event.path, event.path[:-1],)
cur_box_folder = self.traverse_path(client, event, cur_box_folder, folders_to_traverse)
last_dir = os.path.split(event.path)[-1]
if not is_base:
AssertionError(cur_box_folder['name'] == last_dir,
cur_box_folder['name'] + 'not equals ' + last_dir)
event_was_for_dir = 'IN_ISDIR'.lower() in event.maskname.lower()
num_entries = cur_box_folder['item_collection']['total_count']
limit = 100
for offset in range(0, num_entries, limit):
for entry in cur_box_folder.get_items(offset=offset, limit=limit):
if not event_was_for_dir and entry['type'] == 'file' and entry['name'] == event.name:
if entry['id'] not in self.files_from_box:
cur_file = client.file(file_id=entry['id']).get()
if cur_file.delete(): # does not actually check correctly...unless not "ok" is false
# del version_info[cur_file['id']]
r_c.delete(redis_key(cur_file['id']))
else:
self.files_from_box.remove(entry['id']) # just wrote if, assuming create event didn't run
break
elif event_was_for_dir and entry['type'] == 'folder' and entry['name'] == event.name:
if entry['id'] not in self.folders_from_box:
self.get_folder(client, entry['id']).delete()
# cur_folder = client.folder(folder_id=entry['id']).get()
# upload_queue.put(partial(cur_folder.update_contents, event.pathname))
else:
self.folders_from_box.remove(entry['id']) # just wrote if, assuming create event didn't run
break
elif operation == 'move':
crate_logger.debug('Doing a move on: {}'.format(event))
src_event, dest_event = event
folders_to_traverse = self.folders_to_traverse(dest_event.path)
crate_logger.debug(folders_to_traverse)
client = Client(self.oauth)
box_folder = client.folder(folder_id='0').get()
cur_box_folder = box_folder
# if we're modifying in root box dir, then we've already found the folder
cur_box_folder = self.traverse_path(client, dest_event, cur_box_folder, folders_to_traverse)
src_folders_to_traverse = self.folders_to_traverse(src_event.path)
src_box_folder = box_folder
src_box_folder = self.traverse_path(client, src_event, src_box_folder, src_folders_to_traverse)
is_rename = src_event.path == dest_event.path
# is_a_directory = 'IN_ISDIR'.lower() in dest_event.maskname.lower()
did_find_src_file = os.path.isdir(dest_event.pathname) # true if we are a directory :)
did_find_src_folder = os.path.isfile(dest_event.pathname) # true if we are a regular file :)
is_file = os.path.isfile(dest_event.pathname)
is_dir = os.path.isdir(dest_event.pathname)
move_from_remote = False
src_num_entries = src_box_folder['item_collection']['total_count']
limit = 100
for offset in range(0, src_num_entries, limit):
for entry in src_box_folder.get_items(offset=offset, limit=limit):
did_find_src_file = is_file and entry['name'] == src_event.name and entry['type'] == 'file'
did_find_src_folder = is_dir and entry['name'] == src_event.name and entry['type'] == 'folder'
if did_find_src_file:
src_file = client.file(file_id=entry['id']).get()
if is_rename:
src_file.rename(dest_event.name)
else:
did_find_cur_file = os.path.isdir(dest_event.pathname) # should check box instead
did_find_cur_folder = os.path.isfile(dest_event.pathname) # should check box instead
cur_num_entries = cur_box_folder['item_collection']['total_count']
for cur_offset in range(0, cur_num_entries, limit):
for cur_entry in cur_box_folder.get_items(offset=cur_offset, limit=limit):
matching_name = cur_entry['name'] == dest_event.name
did_find_cur_file = is_file and matching_name and isinstance(cur_entry, File)
did_find_cur_folder = is_dir and matching_name and isinstance(cur_entry, Folder)
if did_find_cur_file:
self.upload_queue.put([os.path.getmtime(dest_event.pathname),
partial(cur_entry.update_contents,
dest_event.pathname),
self.oauth])
self.upload_queue.put(partial(src_file.delete))
break
elif did_find_cur_folder:
crate_logger.debug(
'do not currently support movinga same name folder into parent with'
'folder inside of the same name -- would may need to update the '
'contents')
break
if (is_file and did_find_cur_file) or (is_dir and did_find_cur_folder):
break
if is_file and not did_find_cur_file:
src_file.move(cur_box_folder)
# do not yet support moving and renaming in one go
assert src_file['name'] == dest_event.name
elif did_find_src_folder:
src_folder = client.folder(folder_id=entry['id']).get()
if is_rename:
src_folder.rename(dest_event.name)
else:
src_folder.move(cur_box_folder)
# do not yet support moving and renaming in one go
assert src_folder['name'] == dest_event.name
elif entry['name'] == dest_event.name:
move_from_remote = True
if not move_from_remote: # if it was moved from a different folder on remote, could be false still
dest_box_folder = box_folder
dest_folders_to_traverse = self.folders_to_traverse(dest_event.path)
dest_box_folder = self.traverse_path(client, dest_event, dest_box_folder, dest_folders_to_traverse)
dest_num_entries = dest_box_folder['item_collection']['total_count']
limit = 100
for offset in range(0, dest_num_entries, limit):
for entry in cur_box_folder.get_items(offset=offset, limit=limit):
if entry['name'] == dest_event.name:
move_from_remote = True
break
if not move_from_remote:
if is_file and not did_find_src_file:
# src file [should] no longer exist[s]. this file did not originate in box, too.
last_modified_time = os.path.getmtime(dest_event.pathname)
self.upload_queue.put([last_modified_time,
partial(cur_box_folder.upload, dest_event.pathname, dest_event.name),
self.oauth])
elif is_dir and not did_find_src_folder:
self.upload_queue.put(partial(cur_box_folder.create_subfolder, dest_event.name))
wm.add_watch(dest_event.pathname, rec=True, mask=mask)
elif operation == 'create':
crate_logger.debug("Creating: {}".format(event.pathname))
folders_to_traverse = self.folders_to_traverse(event.path)
crate_logger.debug(folders_to_traverse)
client = Client(self.oauth)
box_folder = client.folder(folder_id='0').get()
cur_box_folder = box_folder
# if we're modifying in root box dir, then we've already found the folder
is_base = BOX_DIR in (event.path, event.path[:-1],)
cur_box_folder = self.traverse_path(client, event, cur_box_folder, folders_to_traverse)
last_dir = os.path.split(event.path)[-1]
if not is_base:
assert cur_box_folder['name'] == last_dir
did_find_the_file = os.path.isdir(event.pathname) # true if we are a directory :)
did_find_the_folder = os.path.isfile(event.pathname) # true if we are a regular file :)
is_file = os.path.isfile(event.pathname)
is_dir = os.path.isdir(event.pathname)
num_entries = cur_box_folder['item_collection']['total_count']
limit = 100
for offset in range(0, num_entries, limit):
for entry in cur_box_folder.get_items(offset=offset, limit=limit):
did_find_the_file = is_file and entry['type'] == 'file' and entry['name'] == event.name
did_find_the_folder = is_dir and entry['type'] == 'folder' and entry['name'] == event.name
if did_find_the_file:
if entry['id'] not in self.files_from_box:
# more accurately, was this created offline?
AssertionError(False,
'We should not be able to create a '
'file that exists in box; should be a close/modify.')
crate_logger.debug('Update the file: {}'.format(event.pathname))
a_file = client.file(file_id=entry['id']).get()
# seem it is possible to get more than one create (without having a delete in between)
self.upload_queue.put(partial(a_file.update_contents, event.pathname))
# cur_box_folder.upload(event.pathname, event.name)
else:
self.files_from_box.remove(entry['id']) # just downloaded it
break
elif did_find_the_folder:
# we are not going to re-create the folder, but we are also not checking if the contents in this
# local creation are different from the contents in box.
if entry['id'] in self.folders_from_box:
self.folders_from_box.remove(entry['id']) # just downloaded it
break
if is_file and not did_find_the_file:
crate_logger.debug('Upload the file: {}'.format(event.pathname))
last_modified_time = os.path.getctime(event.pathname)
self.upload_queue.put([last_modified_time, partial(cur_box_folder.upload, event.pathname, event.name),
self.oauth])
elif is_dir and not did_find_the_folder:
crate_logger.debug('Upload the folder: {}'.format(event.pathname))
self.upload_queue.put(partial(cur_box_folder.create_subfolder, event.name))
wm.add_watch(event.pathname, rec=True, mask=mask)
elif operation == 'modify':
crate_logger.debug("{op}...: {pathname}".format(op=operation, pathname=event.pathname))
folders_to_traverse = self.folders_to_traverse(event.path)
crate_logger.debug(folders_to_traverse)
client = Client(self.oauth)
cur_box_folder = None
folder_id = '0'
retry_limit = 5
cur_box_folder = get_box_folder(client, cur_box_folder, folder_id, retry_limit)
# if we're modifying in root box dir, then we've already found the folder
is_base = BOX_DIR in (event.path, event.path[:-1],)
cur_box_folder = self.traverse_path(client, event, cur_box_folder, folders_to_traverse)
last_dir = os.path.split(event.path)[-1]
if not is_base:
AssertionError(cur_box_folder['name'] == last_dir,
cur_box_folder['name'] + 'not equals ' + last_dir)
did_find_the_file = os.path.isdir(event.pathname) # true if we are a directory :)
did_find_the_folder = os.path.isfile(event.pathname) # true if we are a regular file :)
is_file = os.path.isfile(event.pathname)
is_dir = os.path.isdir(event.pathname)
num_entries = cur_box_folder['item_collection']['total_count']
limit = 100
for offset in range(0, num_entries, limit):
for entry in cur_box_folder.get_items(offset=offset, limit=limit):
did_find_the_file = is_file and entry['type'] == 'file' and entry['name'] == event.name
did_find_the_folder = is_dir and entry['type'] == 'folder' and entry['name'] == event.name
if did_find_the_file:
last_modified_time = os.path.getmtime(event.pathname)
if entry['id'] not in self.files_from_box:
cur_file = client.file(file_id=entry['id']).get()
can_update = True
was_versioned = r_c.exists(redis_key(cur_file['id']))
try:
info = redis_get(r_c, cur_file) if was_versioned else None
info = info if was_versioned else {'fresh_download': True,
'etag': '0', 'time_stamp': 0}
item_version = info
if cur_file['etag'] == item_version['etag'] and \
((item_version['fresh_download'] and item_version[
'time_stamp'] >= last_modified_time) or
(not item_version['fresh_download'] and item_version[
'time_stamp'] >= last_modified_time)):
can_update = False
if can_update:
self.upload_queue.put([last_modified_time,
partial(cur_file.update_contents, event.pathname),
self.oauth])
else:
is_new_time_stamp = item_version['time_stamp'] >= last_modified_time
crate_logger.debug('Skipping the update because not versioned: {not_versioned}, '
'fresh_download: {fresh_download}, '
'version time_stamp >= '
'new time stamp: {new_time_stamp}, '
'event pathname: {path_name}, '
'cur file id: {obj_id}'.format(not_versioned=not was_versioned,
fresh_download=item_version[
'fresh_download'],
new_time_stamp=is_new_time_stamp,
path_name=event.pathname,
obj_id=cur_file['id']))
except TypeError:
crate_logger.debug(traceback.format_exc())
except Exception:
crate_logger.debug(traceback.format_exc())
else:
self.files_from_box.remove(entry['id']) # just wrote if, assuming create event didn't run
break
elif did_find_the_folder:
if entry['id'] not in self.folders_from_box:
crate_logger.debug('Cannot create a subfolder when it already exists: {}'.format(event.pathname))
# cur_folder = client.folder(folder_id=entry['id']).get()
# upload_queue.put(partial(cur_folder.update_contents, event.pathname))
else:
self.folders_from_box.remove(entry['id']) # just wrote if, assuming create event didn't run
break
if is_file and not did_find_the_file:
crate_logger.debug('Uploading contents...: {}'.format(event.pathname))
last_modified_time = os.path.getmtime(event.pathname)
self.upload_queue.put([last_modified_time,
partial(cur_box_folder.upload, event.pathname, event.name),
self.oauth])
if is_dir and not did_find_the_folder:
crate_logger.debug('Creating a sub-folder...: {}'.format(event.pathname))
self.upload_queue.put(partial(cur_box_folder.create_subfolder, event.name))
wm.add_watch(event.pathname, rec=True, mask=mask)
elif operation == 'real_close':
crate_logger.debug("Real close...: {}".format(event.pathname))
folders_to_traverse = self.folders_to_traverse(event.path)
crate_logger.debug(folders_to_traverse)
client = Client(self.oauth)
cur_box_folder = None
cur_box_folder = get_box_folder(client, cur_box_folder, '0', 5)
# if we're modifying in root box dir, then we've already found the folder
is_base = BOX_DIR in (event.path, event.path[:-1],)
cur_box_folder = self.traverse_path(client, event, cur_box_folder, folders_to_traverse)
last_dir = os.path.split(event.path)[-1]
if not is_base:
AssertionError(cur_box_folder['name'] == last_dir,
cur_box_folder['name'] + 'not equals ' + last_dir)
did_find_the_file = os.path.isdir(event.pathname) # true if we are a directory :)
did_find_the_folder = os.path.isfile(event.pathname) # true if we are a regular file :)
is_file = os.path.isfile(event.pathname)
is_dir = os.path.isdir(event.pathname)
num_entries = cur_box_folder['item_collection']['total_count']
limit = 100
for offset in range(0, num_entries, limit):
for entry in cur_box_folder.get_items(offset=offset, limit=limit):
did_find_the_file = is_file and entry['type'] == 'file' and entry['name'] == event.name
did_find_the_folder = is_dir and entry['type'] == 'folder' and entry['name'] == event.name
if did_find_the_file:
break
# not a box file/folder (though could have been copied from a local box item)
if is_file and not did_find_the_file:
last_modified_time = os.path.getmtime(event.pathname)
self.upload_queue.put([last_modified_time,
partial(cur_box_folder.upload, event.pathname, event.name),
self.oauth])
elif is_dir and not did_find_the_folder:
cur_box_folder.create_subfolder(event.name)
wm.add_watch(event.pathname, rec=True, mask=mask, auto_add=True)
from boxsdk import OAuth2
from boxsdk import Client
import argparse
import os
oauth = OAuth2(
client_id='bde3fxtg8ysjbrtdhlflftc1u9brsnbl',
client_secret='jxfAFzhTdPA2DXBAIXyz4fIPl4OjzwAR',
access_token='Sri58hN43NPONxezfk74vJgVeLlNSmyv',
)
client = Client(oauth)
root_folder = client.folder(folder_id='0')
# print('root_folder_with_info.name:' + root_folder_with_info.name)
parser = argparse.ArgumentParser(description='download a file from Box')
parser.add_argument('--file', '-f', help='file path')
parser.add_argument('--dstpath', '-d', help='destination path')
args = parser.parse_args()
if not args.file:
print("Please specify the file" + "Usage: upload.py -f file/path -d Box/path")
exit(1)
dwn_file = args.file
dst_file = args.dstpath
# download
def download(filedata):
if args.dstpath:
if os.path.isdir(dst_file):
class BoxInstance(object):
""" class to make use of google python api """
def __init__(self, number_to_process=-1, credential_file=HOMEDIR + '/.box/credentials'):
""" init function """
self.credential_file = credential_file
self.redirect_uri = ''
self.client_id = ''
self.client_secret = ''
self.list_of_keys = {}
self.list_of_mimetypes = {}
self.items_processed = 0
self.list_of_folders = {}
self.list_of_items = {}
self.number_to_process = number_to_process
self.read_credentials()
self.client = self.get_auth()
def store_tokens(self, access_token, refresh_token):
with open(os.path.join(HOMEDIR, '.box_tokens.pkl'), 'w') as credfile:
tmp = (access_token, refresh_token)
pickle.dump(obj=tmp, file=credfile, protocol=pickle.HIGHEST_PROTOCOL)
def read_credentials(self, credential_file=HOMEDIR + '/.box/credentials'):
""" read credentials from file """
with open(credential_file, 'r') as credfile:
for line in credfile:
key_, val_ = line.split()[:2]
for key in ('redirect_uri', 'client_id', 'client_secret'):
if key.lower() == key_.strip().lower():
setattr(self, key, val_)
def get_auth(self):
""" do authorization """
if os.path.exists(os.path.join(HOMEDIR, '.box_tokens.pkl')):
with open(os.path.join(HOMEDIR, '.box_tokens.pkl'), 'rb') as pfile:
self.access_token, self.refresh_token = pickle.load(pfile)
self.oauth = OAuth2(
client_id=self.client_id,
client_secret=self.client_secret,
store_tokens=self.store_tokens,
access_token=self.access_token,
refresh_token=self.refresh_token)
else:
self.oauth = OAuth2(
client_id=self.client_id,
client_secret=self.client_secret,
store_tokens=self.store_tokens)
auth_url, csrf_token = self.oauth.get_authorization_url(self.redirect_uri)
code = get_auth_code(auth_url, self.redirect_uri)
print(code)
self.access_token, self.refresh_token = \
self.oauth.authenticate(code)
self.client = Client(self.oauth)
return self.client
def list_files(self, callback_fn, number_to_process=-1):
""" list non-directory files """
fields = [
'id', 'size', 'etag', 'description', 'parent', 'name', 'type', 'modified_at', 'sha1'
]
number_complete = {'count': 0}
def walk_nodes(parentid='0'):
parent_node = self.client.folder(folder_id=parentid).get()
cur_offset = 0
while True:
new_items = parent_node.get_items(limit=100, offset=cur_offset, fields=fields)
if not new_items:
break
for item in new_items:
if number_to_process > 0 \
and number_complete['count'] > number_to_process:
break
number_complete['count'] += 1
item = item._response_object
item['parentid'] = parentid
if item.get('type', '') == 'folder':
walk_nodes(parentid=item['id'])
else:
callback_fn(item)
#print(parent_node._response_object['name'], cur_offset)
cur_offset += 100
walk_nodes(parentid='0')
def get_folders(self, callback_fn, number_to_process=-1):
""" get folders """
number_complete = {'count': 0}
def walk_nodes(parentid='0'):
parent_node = self.client.folder(folder_id=parentid).get()
item_col = parent_node._response_object.get('item_collection', {})
entries = item_col.get('entries', [])
for item in entries:
item['parentid'] = parentid
if item.get('type', '') == 'folder':
if number_to_process > 0 \
and number_complete['count'] > number_to_process:
return
number_complete['count'] += 1
node = self.client.folder(folder_id=item['id']).get()
node = node._response_object
node['parentid'] = item['parentid']
callback_fn(node)
walk_nodes(parentid=item['id'])
walk_nodes(parentid='0')
def download(self, did, exportfile, sha1sum=None):
""" download using dlink url """
dirname = os.path.dirname(os.path.abspath(exportfile))
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(exportfile + '.new', 'w') as outfile:
self.client.file(file_id=did).download_to(outfile)
if sha1sum:
from sync_app.util import get_sha1
sha = get_sha1(exportfile + '.new')
if sha != sha1sum:
raise TypeError('%s %s' % (sha, sha1sum))
os.rename('%s.new' % exportfile, exportfile)
return True
def upload(self, fname, parent_id='0'):
""" upload fname and assign parent_id if provided """
bname = os.path.basename(fname)
parent = self.client.folder(folder_id=parent_id)
try:
file_obj = parent.upload(file_path=fname, file_name=bname).get()
except BoxAPIException as exc:
print('BoxAPIException upload %s' % exc)
raise
item = file_obj._response_object
item['parentid'] = parent_id
return item
def create_directory(self, dname, parent_id='0'):
""" create directory, assign parent_id if supplied """
if not parent_id:
raise ValueError('need to specify parent_id')
parent = self.client.folder(folder_id=parent_id)
try:
parent.create_subfolder(dname)
except BoxAPIException as exc:
print('create_directory BoxAPIException %s %s' % (dname, exc))
pass
parent = parent.get()
item = parent._response_object
items = item.get('item_collection', {}).get('entries', [])
for item in items:
if item['type'] == 'folder' and item['name'] == dname:
item['parentid'] = parent_id
return item
def delete_directory(self, dirid):
""" delete directory by folderid """
return self.client.folder(folder_id=dirid).delete()
def delete_file(self, fileid):
""" delete file by fileid """
return self.client.file(file_id=fileid).delete()
from boxsdk import OAuth2
from boxsdk import Client
import io
from boxsdk.exception import BoxAPIException
oauth = OAuth2(
client_id='bde3fxtg8ysjbrtdhlflftc1u9brsnbl',
client_secret='jxfAFzhTdPA2DXBAIXyz4fIPl4OjzwAR',
access_token='Zd3HyETzTvivOXZFqksBbIgZZlWrbgMe',
)
client = Client(oauth)
root_folder = client.folder(folder_id='0')
root_folder_with_info = root_folder.get()
# shared_folder = root_folder.create_subfolder('shared_folder')
# uploaded_file = shared_folder.upload('test.txt')
# shared_link = shared_folder.get_shared_link()
print('root_folder_with_info.name:' + root_folder_with_info.name)
root_folder_with_limited_info = root_folder.get(fields=['owned_by'])
print(root_folder_with_limited_info.owned_by)
# print('root_folder_with_limited_info:' + root_folder_with_limited_info.owned_by)
folder_info = client.folder(folder_id='me')
print(folder_info)
print(client.file(file_id='me'))
me = client.user(user_id='me').get()
print(me)
print('name:' + me.name)
print('login:' + me.login)
