def url(self, destination):
if isinstance(destination, IOHandler):
output_name, _ = _build_output_name(destination)
just_file_name = os.path.basename(output_name)
dst = urljoin(str(destination.uuid), just_file_name)
else:
dst = destination
# make sure base url ends with '/'
baseurl = self.output_url.rstrip('/') + '/'
url = urljoin(baseurl, dst)
return url
def store(self, output):
import shutil, tempfile, math
file_name = output.file
file_block_size = os.stat(file_name).st_blksize
avail_size = get_free_space(self.target)
file_size = os.stat(file_name).st_size
# calculate space used according to block size
actual_file_size = math.ceil(
file_size / float(file_block_size)) * file_block_size
if avail_size < actual_file_size:
raise NotEnoughStorage('Not enough space in %s to store %s' %
(self.target, file_name))
(prefix, suffix) = os.path.splitext(file_name)
if not suffix:
suffix = output.output_format.get_extension()
(file_dir, file_name) = os.path.split(prefix)
output_name = tempfile.mkstemp(suffix=suffix,
prefix=file_name,
dir=self.target)[1]
full_output_name = os.path.join(self.target, output_name)
LOGGER.info('Storing file output to %s', full_output_name)
shutil.copy2(output.file, full_output_name)
just_file_name = os.path.basename(output_name)
url = urljoin(self.output_url, just_file_name)
LOGGER.info('File output URI: %s', url)
return (STORE_TYPE.PATH, output_name, url)
def store(self, output):
import shutil, tempfile
file_name = output.file
try:
import ftplib
except:
raise StorageDependecyError(
'[400]:Dependecny for FTP Storage not met')
(prefix, suffix) = os.path.splitext(file_name)
if not suffix:
suffix = output.output_format.get_extension()
(file_dir, file_name) = os.path.split(prefix)
output_name = tempfile.mkstemp(suffix=suffix,
prefix=file_name,
dir=self.target)[1]
full_output_name = os.path.join(self.target, output_name)
# LOGGER.info('Storing file output to %s', full_output_name)
shutil.copy2(output.file, full_output_name)
ftp = ftplib.FTP(self.ftp_host)
try:
ftp.login(self.ftp_user, self.ftp_password)
except:
raise StorageAuthenticationError('Invalid FTP credentials')
ftp.storlines("STOR " + output.file, open(full_output_name))
just_file_name = os.path.basename(output_name)
url = urljoin(self.ftp_host, just_file_name)
LOGGER.info('File Saved in FTP Server: %s', url)
return (STORE_TYPE.FTP, output_name, url)
def store(self, output):
import math
import shutil
import tempfile
file_name = output.file
file_block_size = os.stat(file_name).st_blksize
# get_free_space delivers the numer of free blocks, not the available size!
avail_size = get_free_space(self.target) * file_block_size
file_size = os.stat(file_name).st_size
# calculate space used according to block size
actual_file_size = math.ceil(file_size / float(file_block_size)) * file_block_size
if avail_size < actual_file_size:
raise NotEnoughStorage('Not enough space in %s to store %s' % (self.target, file_name))
(prefix, suffix) = os.path.splitext(file_name)
if not suffix:
suffix = output.output_format.extension
(file_dir, file_name) = os.path.split(prefix)
output_name = tempfile.mkstemp(suffix=suffix, prefix=file_name,
dir=self.target)[1]
full_output_name = os.path.join(self.target, output_name)
LOGGER.info('Storing file output to %s', full_output_name)
shutil.copy2(output.file, full_output_name)
just_file_name = os.path.basename(output_name)
url = urljoin(self.output_url, just_file_name)
LOGGER.info('File output URI: %s', url)
return (STORE_TYPE.PATH, output_name, url)
def store(self, output):
import shutil, tempfile, math
file_name = output.file
file_block_size = os.stat(file_name).st_blksize
avail_size = get_free_space(self.target)
file_size = os.stat(file_name).st_size
# calculate space used according to block size
actual_file_size = math.ceil(file_size / float(file_block_size)) * file_block_size
if avail_size < actual_file_size:
raise NotEnoughStorage('Not enough space in %s to store %s' % (self.target, file_name))
(prefix, suffix) = os.path.splitext(file_name)
if not suffix:
suffix = output.output_format.get_extension()
(file_dir, file_name) = os.path.split(prefix)
output_name = tempfile.mkstemp(suffix=suffix, prefix=file_name,
dir=self.target)[1]
shutil.copy2(output.file, os.path.join(self.target, output_name))
just_file_name = os.path.basename(output_name)
url = urljoin(self.output_url, just_file_name)
return (STORE_TYPE.PATH, output_name, url)
def store(self, output):
import shutil, tempfile
file_name = output.file
try:
import ftplib
except:
raise StorageDependecyError('[400]:Dependecny for FTP Storage not met')
(prefix, suffix) = os.path.splitext(file_name)
if not suffix:
suffix = output.output_format.get_extension()
(file_dir, file_name) = os.path.split(prefix)
output_name = tempfile.mkstemp(suffix=suffix, prefix=file_name,
dir=self.target)[1]
full_output_name = os.path.join(self.target, output_name)
# LOGGER.info('Storing file output to %s', full_output_name)
shutil.copy2(output.file, full_output_name)
ftp = ftplib.FTP(self.ftp_host)
try:
ftp.login(self.ftp_user, self.ftp_password)
except:
raise StorageAuthenticationError('Invalid FTP credentials')
ftp.storlines("STOR " + output.file, open(full_output_name))
just_file_name = os.path.basename(output_name)
url = urljoin(self.ftp_host, just_file_name)
LOGGER.info('File Saved in FTP Server: %s', url)
return (STORE_TYPE.FTP, output_name, url)
def store(self, output):
import math
import shutil
import tempfile
import uuid
file_name = output.file
request_uuid = output.uuid or uuid.uuid1()
file_block_size = os.stat(file_name).st_blksize
# get_free_space delivers the numer of free blocks, not the available size!
avail_size = get_free_space(self.target) * file_block_size
file_size = os.stat(file_name).st_size
# calculate space used according to block size
actual_file_size = math.ceil(
file_size / float(file_block_size)) * file_block_size
if avail_size < actual_file_size:
raise NotEnoughStorage('Not enough space in {} to store {}'.format(
self.target, file_name))
# create a target folder for each request
target = os.path.join(self.target, str(request_uuid))
if not os.path.exists(target):
os.makedirs(target)
# build output name
(prefix, suffix) = os.path.splitext(file_name)
if not suffix:
suffix = output.data_format.extension
(file_dir, file_name) = os.path.split(prefix)
output_name = file_name + suffix
# build tempfile in case of duplicates
if os.path.exists(os.path.join(target, output_name)):
output_name = tempfile.mkstemp(suffix=suffix,
prefix=file_name + '_',
dir=target)[1]
full_output_name = os.path.join(target, output_name)
LOGGER.info('Storing file output to %s', full_output_name)
shutil.copy2(output.file, full_output_name)
just_file_name = os.path.basename(output_name)
# make sure base url ends with '/'
baseurl = self.output_url.rstrip('/') + '/'
baseurl += str(request_uuid) + '/'
url = urljoin(baseurl, just_file_name)
LOGGER.info('File output URI: %s', url)
return (STORE_TYPE.PATH, output_name, url)
def _do_store(self, output):
import math
import shutil
import tempfile
import uuid
file_name = output.file
request_uuid = output.uuid or uuid.uuid1()
file_block_size = os.stat(file_name).st_blksize
# get_free_space delivers the numer of free blocks, not the available size!
avail_size = get_free_space(self.target) * file_block_size
file_size = os.stat(file_name).st_size
# calculate space used according to block size
actual_file_size = math.ceil(file_size / float(file_block_size)) * file_block_size
if avail_size < actual_file_size:
raise NotEnoughStorage('Not enough space in {} to store {}'.format(self.target, file_name))
# create a target folder for each request
target = os.path.join(self.target, str(request_uuid))
if not os.path.exists(target):
os.makedirs(target)
# build output name
(prefix, suffix) = os.path.splitext(file_name)
if not suffix:
suffix = output.data_format.extension
(file_dir, file_name) = os.path.split(prefix)
output_name = file_name + suffix
# build tempfile in case of duplicates
if os.path.exists(os.path.join(target, output_name)):
output_name = tempfile.mkstemp(suffix=suffix, prefix=file_name + '_',
dir=target)[1]
full_output_name = os.path.join(target, output_name)
LOGGER.info('Storing file output to {}'.format(full_output_name))
shutil.copy2(output.file, full_output_name)
just_file_name = os.path.basename(output_name)
# make sure base url ends with '/'
baseurl = self.output_url.rstrip('/') + '/'
baseurl += str(request_uuid) + '/'
url = urljoin(baseurl, just_file_name)
LOGGER.info('File output URI: {}'.format(url))
return (STORE_TYPE.PATH, output_name, url)
def _do_store(self, output):
import platform
import math
import shutil
import tempfile
import uuid
file_name = output.file
request_uuid = output.uuid or uuid.uuid1()
# st.blksize is not available in windows, skips the validation on windows
if platform.system() != 'Windows':
file_block_size = os.stat(file_name).st_blksize
# get_free_space delivers the numer of free blocks, not the available size!
avail_size = get_free_space(self.target) * file_block_size
file_size = os.stat(file_name).st_size
# calculate space used according to block size
actual_file_size = math.ceil(
file_size / float(file_block_size)) * file_block_size
if avail_size < actual_file_size:
raise NotEnoughStorage(
'Not enough space in {} to store {}'.format(
self.target, file_name))
# create a target folder for each request
target = os.path.join(self.target, str(request_uuid))
if not os.path.exists(target):
os.makedirs(target)
# build output name
output_name, suffix = _build_output_name(output)
# build tempfile in case of duplicates
if os.path.exists(os.path.join(target, output_name)):
output_name = tempfile.mkstemp(suffix=suffix,
prefix=file_name + '_',
dir=target)[1]
full_output_name = os.path.join(target, output_name)
LOGGER.info('Storing file output to {}'.format(full_output_name))
shutil.copy2(output.file, full_output_name)
just_file_name = os.path.basename(output_name)
url = self.url(urljoin(str(request_uuid), just_file_name))
LOGGER.info('File output URI: {}'.format(url))
return (STORE_TYPE.PATH, output_name, url)
def store(self, output):
import shutil, tempfile, httplib2
try:
from oauth2client.service_account import ServiceAccountCredentials
from apiclient.discovery import build
except:
raise StorageDependecyError(
'[400]:Dependecny for Google Drive Storage not met')
file_name = output.file
(prefix, suffix) = os.path.splitext(file_name)
if not suffix:
suffix = output.output_format.get_extension()
(file_dir, file_name) = os.path.split(prefix)
output_name = tempfile.mkstemp(suffix=suffix,
prefix=file_name,
dir=self.target)[1]
full_output_name = os.path.join(self.target, output_name)
# LOGGER.info('Storing file output to %s', full_output_name)
shutil.copy2(output.file, full_output_name)
just_file_name = os.path.basename(output_name)
local_url = urljoin(self.output_url, just_file_name)
scopes = 'https://www.googleapis.com/auth/drive.file'
credentials = ServiceAccountCredentials.from_json_keyfile_name(
'service-account.json', scopes)
http = credentials.authorize(httplib2.Http())
drive_service = build('drive', 'v3', http=http)
file_metadata = {
'name': file_name,
'mimeType': 'application/vnd.google-apps.file'
}
media = MediaFileUpload(local_url, mimetype='text/csv', resumable=True)
drive_service = build('drive', 'v3', http=http)
uploaded_file = drive_service.files().create(body=file_metadata,
media_body=media,
fields='id').execute()
file_id = uploaded_file.get('id')
new_permission = {'type': 'anyone', 'role': 'writer', 'withLink': True}
permission = drive_service.permissions().create(
fileId=file_id, body=new_permission).execute()
download_file = drive_service.files().get(fileId=file_id).execute()
url = download_file['webContentLink']
return (STORE_TYPE.GOOGLEDRIVE, output_name, url)
def store(self, output):
import shutil, tempfile, httplib2
try:
from oauth2client.service_account import ServiceAccountCredentials
from apiclient.discovery import build
except:
raise StorageDependecyError('[400]:Dependecny for Google Drive Storage not met')
file_name = output.file
(prefix, suffix) = os.path.splitext(file_name)
if not suffix:
suffix = output.output_format.get_extension()
(file_dir, file_name) = os.path.split(prefix)
output_name = tempfile.mkstemp(suffix=suffix, prefix=file_name,
dir=self.target)[1]
full_output_name = os.path.join(self.target, output_name)
# LOGGER.info('Storing file output to %s', full_output_name)
shutil.copy2(output.file, full_output_name)
just_file_name = os.path.basename(output_name)
local_url = urljoin(self.output_url, just_file_name)
scopes = 'https://www.googleapis.com/auth/drive.file'
credentials = ServiceAccountCredentials.from_json_keyfile_name('service-account.json', scopes)
http = credentials.authorize(httplib2.Http())
drive_service = build('drive', 'v3', http=http)
file_metadata = { 'name' : file_name,'mimeType' : 'application/vnd.google-apps.file'}
media = MediaFileUpload(local_url,mimetype='text/csv',resumable=True)
drive_service = build('drive', 'v3', http=http)
uploaded_file = drive_service.files().create(body=file_metadata,media_body=media,fields='id').execute()
file_id = uploaded_file.get('id')
new_permission = {'type':'anyone','role':'writer','withLink':True}
permission = drive_service.permissions().create(fileId=file_id,body=new_permission).execute()
download_file = drive_service.files().get(fileId=file_id).execute()
url = download_file['webContentLink']
return (STORE_TYPE.GOOGLEDRIVE, output_name, url)
def store(self, output):
import math
import shutil
import tempfile
file_name = output.file
file_block_size = os.stat(file_name).st_blksize
# get_free_space delivers the numer of free blocks, not the available size!
avail_size = get_free_space(self.target) * file_block_size
file_size = os.stat(file_name).st_size
# calculate space used according to block size
actual_file_size = math.ceil(
file_size / float(file_block_size)) * file_block_size
if avail_size < actual_file_size:
raise NotEnoughStorage('Not enough space in %s to store %s' %
(self.target, file_name))
(prefix, suffix) = os.path.splitext(file_name)
if not suffix:
suffix = output.output_format.extension
(file_dir, file_name) = os.path.split(prefix)
output_name = tempfile.mkstemp(suffix=suffix,
prefix=file_name,
dir=self.target)[1]
full_output_name = os.path.join(self.target, output_name)
LOGGER.info('Storing file output to %s', full_output_name)
shutil.copy2(output.file, full_output_name)
just_file_name = os.path.basename(output_name)
# make sure base url ends with '/'
baseurl = self.output_url.rstrip('/') + '/'
url = urljoin(baseurl, just_file_name)
LOGGER.info('File output URI: %s', url)
return (STORE_TYPE.PATH, output_name, url)