def push(self, fp, storeurl, uniq=None):
"Push a local file (file pointer) to the store"
log.debug('local.push: url=%s', storeurl)
origpath = localpath = url2localpath(storeurl)
fpath,ext = os.path.splitext(origpath)
_mkdir (os.path.dirname(localpath))
uniq = uniq or make_uniq_code()
for x in xrange(len(uniq)-7):
#for x in range(100):
if not os.path.exists (localpath):
log.debug('local.write: %s -> %s' , tounicode(storeurl), tounicode(localpath))
#patch for no copy file uploads - check for regular file or file like object
try:
move_file (fp, localpath)
except OSError as e:
if not os.path.exists (localpath):
log.exception ("Problem moving file to%s ", localpath)
else:
log.error ("Problem moving file, but it seems to be there.. check permissions on store")
#log.debug ("local.push: top = %s path= %s",self.top_path, localpath )
ident = localpath[len(self.top_path):]
#if ident[0] == '/':
# ident = ident[1:]
ident = localpath2url(ident)
log.info('local push blob_id: %s -> %s', tounicode(ident), tounicode(localpath))
return ident, localpath
localpath = "%s-%s%s" % (fpath , uniq[3:7+x] , ext)
#localpath = "%s-%04d%s" % (fpath , x , ext)
log.warn ("local.write: File exists... trying %s", tounicode(localpath))
raise DuplicateFile(localpath)
def __init__(self, resource_type = None, parent = None, owner_id = None, mex_id = None):
"""Create a taggable resource : not usually called directly
@param resource_type: A string type of the new resource i.e. user, project etc.
@param parent: A parent of the resource or None
@owner_id: An integer ID, a BQUser, current user if None, False : don't set
@mex_id: An integer ID, A ModuleExecution, current mex if None, or False : don't set
"""
if resource_type is None:
resource_type = self.xmltag
self.resource_type = resource_type
# By defualt you are the document
if parent:
parent.children.append(self)
self.document = parent.document
self.perm = parent.perm
self.hidden = parent.hidden
else:
self.document = self
self.resource_uniq = make_uniq_code()
self.perm = PRIVATE
#self.hidden = None
#if self.resource_type == 'mex':
# self.mex = self
# session['mex'] = self
#if self.resource_type == 'user':
# self.owner = self
self.ts = datetime.now()
#log.debug("new taggable user:"******"mex_id = %s" , mex_id)
if mex_id is not None:
log.debug ("setting mex_id %s" , mex_id)
self.mex_id = mex_id
if owner_id is not False:
owner_id = owner_id or identity.get_user_id()
if owner_id is not None:
if isinstance(owner_id, Taggable):
self.owner = owner_id
else:
self.owner_id = owner_id
else:
log.warn ("CREATING taggable %s with no owner" , str(self) )
admin = identity.get_admin()
if admin:
log.warn("Setting owner to admin")
self.owner_id = admin.id
def upgrade():
"Ensure all top-level objects have resource_uniq and tag all resource_unique with generation code"
cntxt = context.get_context()
SessionMaker = sessionmaker(bind=cntxt.bind)
# toplevel = DBSession.query(Taggable).filter(Taggable.resource_parent_id == None, Taggable.resource_uniq == None)
# for resource in toplevel:
# uid = make_short_uuid()
# #resource.resource_uniq = "00-%s" % uid
# print resource.resource_type, uid
# resource.resource_uniq = "00-%s" % uid
Base = declarative_base()
metadata = MetaData(bind=cntxt.bind)
class Taggable(Base):
__table__ = Table('taggable', metadata, autoload=True)
DBSession = SessionMaker()
toplevel = DBSession.query(Taggable).filter(
Taggable.resource_parent_id == None)
for resource in toplevel:
if resource.resource_uniq is None:
uid = make_uniq_code()
resource.resource_uniq = uid
elif resource.resource_uniq.startswith('00-'):
continue
elif len(resource.resource_uniq) == 40:
uid = make_uniq_code()
resource.resource_uniq = uid
else:
resource.resource_uniq = "00-%s" % resource.resource_uniq
print "updating %s -> %s" % (resource.id, resource.resource_uniq)
DBSession.commit()
DBSession.close()
def push(self, fp, storeurl, uniq=None):
"Push a local file (file pointer) to the store"
sharename, path = self.split_smb(storeurl)
uniq = uniq or make_uniq_code()
base,ext = os.path.splitext(path)
for x in xrange(len(uniq)-7):
try:
if not self.conn.getAttributes (sharename, path):
break
except smb.OperationError:
path = "%s-%s%s" % (base , uniq[3:7+x] , ext)
written = self.conn.storeFile (sharename, path, fp)
log.debug ("smb wrote %s bytes", written)
return "smb://%s/%s" % (sharename, path), None
def push(self, fp, storeurl, uniq=None):
'write a file to s3'
s3_ident,sub = split_subpath(storeurl)
s3_base,ext = os.path.splitext(s3_ident)
log.info('s3.write: %s -> %s' , storeurl, s3_ident)
uniq = uniq or make_uniq_code()
for x in xrange(len(uniq)-7):
s3_key = s3_ident.replace("s3://","")
if not s3_handler.s3_isfile (self.bucket_id, s3_key, self.creds):
break
s3_ident = "%s-%s%s" % (s3_base , uniq[3:7+x] , ext)
flocal = s3_handler.s3_push_file(fp, self.bucket_id , s3_key, self.cache, self.creds)
flocal = force_filesys (flocal)
return s3_ident, flocal
def push(self, fp, storeurl, uniq = None):
"Push a local file (file pointer) to the store"
log.info('irods.push: %s' , storeurl)
fpath,ext = os.path.splitext(storeurl)
uniq = uniq or make_uniq_code()
try:
for x in xrange(len(uniq)-7):
if not irods.irods_isfile (storeurl, user=self.user, password = self.password):
break
storeurl = "%s-%s%s" % (fpath , uniq[3:7+x] , ext)
flocal = irods.irods_push_file(fp, storeurl, user=self.user, password=self.password, cache=self.cache)
flocal = force_filesys (flocal)
except irods.IrodsError:
log.exception ("During Irods Push")
raise IllegalOperation ("irods push failed")
return storeurl, flocal
def resource_uniq(self, **kw):
'generate a unique code to be used for a resource'
return make_uniq_code()