def push(self, fp, storeurl, uniq=None): "Push a local file (file pointer) to the store" log.debug('local.push: url=%s', storeurl) origpath = localpath = url2localpath(storeurl) fpath,ext = os.path.splitext(origpath) _mkdir (os.path.dirname(localpath)) uniq = uniq or make_uniq_code() for x in xrange(len(uniq)-7): #for x in range(100): if not os.path.exists (localpath): log.debug('local.write: %s -> %s' , tounicode(storeurl), tounicode(localpath)) #patch for no copy file uploads - check for regular file or file like object try: move_file (fp, localpath) except OSError as e: if not os.path.exists (localpath): log.exception ("Problem moving file to%s ", localpath) else: log.error ("Problem moving file, but it seems to be there.. check permissions on store") #log.debug ("local.push: top = %s path= %s",self.top_path, localpath ) ident = localpath[len(self.top_path):] #if ident[0] == '/': # ident = ident[1:] ident = localpath2url(ident) log.info('local push blob_id: %s -> %s', tounicode(ident), tounicode(localpath)) return ident, localpath localpath = "%s-%s%s" % (fpath , uniq[3:7+x] , ext) #localpath = "%s-%04d%s" % (fpath , x , ext) log.warn ("local.write: File exists... trying %s", tounicode(localpath)) raise DuplicateFile(localpath)
def __init__(self, resource_type = None, parent = None, owner_id = None, mex_id = None): """Create a taggable resource : not usually called directly @param resource_type: A string type of the new resource i.e. user, project etc. @param parent: A parent of the resource or None @owner_id: An integer ID, a BQUser, current user if None, False : don't set @mex_id: An integer ID, A ModuleExecution, current mex if None, or False : don't set """ if resource_type is None: resource_type = self.xmltag self.resource_type = resource_type # By defualt you are the document if parent: parent.children.append(self) self.document = parent.document self.perm = parent.perm self.hidden = parent.hidden else: self.document = self self.resource_uniq = make_uniq_code() self.perm = PRIVATE #self.hidden = None #if self.resource_type == 'mex': # self.mex = self # session['mex'] = self #if self.resource_type == 'user': # self.owner = self self.ts = datetime.now() #log.debug("new taggable user:"******"mex_id = %s" , mex_id) if mex_id is not None: log.debug ("setting mex_id %s" , mex_id) self.mex_id = mex_id if owner_id is not False: owner_id = owner_id or identity.get_user_id() if owner_id is not None: if isinstance(owner_id, Taggable): self.owner = owner_id else: self.owner_id = owner_id else: log.warn ("CREATING taggable %s with no owner" , str(self) ) admin = identity.get_admin() if admin: log.warn("Setting owner to admin") self.owner_id = admin.id
def upgrade(): "Ensure all top-level objects have resource_uniq and tag all resource_unique with generation code" cntxt = context.get_context() SessionMaker = sessionmaker(bind=cntxt.bind) # toplevel = DBSession.query(Taggable).filter(Taggable.resource_parent_id == None, Taggable.resource_uniq == None) # for resource in toplevel: # uid = make_short_uuid() # #resource.resource_uniq = "00-%s" % uid # print resource.resource_type, uid # resource.resource_uniq = "00-%s" % uid Base = declarative_base() metadata = MetaData(bind=cntxt.bind) class Taggable(Base): __table__ = Table('taggable', metadata, autoload=True) DBSession = SessionMaker() toplevel = DBSession.query(Taggable).filter( Taggable.resource_parent_id == None) for resource in toplevel: if resource.resource_uniq is None: uid = make_uniq_code() resource.resource_uniq = uid elif resource.resource_uniq.startswith('00-'): continue elif len(resource.resource_uniq) == 40: uid = make_uniq_code() resource.resource_uniq = uid else: resource.resource_uniq = "00-%s" % resource.resource_uniq print "updating %s -> %s" % (resource.id, resource.resource_uniq) DBSession.commit() DBSession.close()
def push(self, fp, storeurl, uniq=None): "Push a local file (file pointer) to the store" sharename, path = self.split_smb(storeurl) uniq = uniq or make_uniq_code() base,ext = os.path.splitext(path) for x in xrange(len(uniq)-7): try: if not self.conn.getAttributes (sharename, path): break except smb.OperationError: path = "%s-%s%s" % (base , uniq[3:7+x] , ext) written = self.conn.storeFile (sharename, path, fp) log.debug ("smb wrote %s bytes", written) return "smb://%s/%s" % (sharename, path), None
def push(self, fp, storeurl, uniq=None): 'write a file to s3' s3_ident,sub = split_subpath(storeurl) s3_base,ext = os.path.splitext(s3_ident) log.info('s3.write: %s -> %s' , storeurl, s3_ident) uniq = uniq or make_uniq_code() for x in xrange(len(uniq)-7): s3_key = s3_ident.replace("s3://","") if not s3_handler.s3_isfile (self.bucket_id, s3_key, self.creds): break s3_ident = "%s-%s%s" % (s3_base , uniq[3:7+x] , ext) flocal = s3_handler.s3_push_file(fp, self.bucket_id , s3_key, self.cache, self.creds) flocal = force_filesys (flocal) return s3_ident, flocal
def push(self, fp, storeurl, uniq = None): "Push a local file (file pointer) to the store" log.info('irods.push: %s' , storeurl) fpath,ext = os.path.splitext(storeurl) uniq = uniq or make_uniq_code() try: for x in xrange(len(uniq)-7): if not irods.irods_isfile (storeurl, user=self.user, password = self.password): break storeurl = "%s-%s%s" % (fpath , uniq[3:7+x] , ext) flocal = irods.irods_push_file(fp, storeurl, user=self.user, password=self.password, cache=self.cache) flocal = force_filesys (flocal) except irods.IrodsError: log.exception ("During Irods Push") raise IllegalOperation ("irods push failed") return storeurl, flocal
def resource_uniq(self, **kw): 'generate a unique code to be used for a resource' return make_uniq_code()