def filerec(self, rel_path, create=False):
'''Return a database record for a file'''
import sqlalchemy.orm.exc
s = self.bundle.database.session
if not rel_path:
raise ValueError('Must supply rel_path')
try:
o = (s.query(File).filter(File.path==rel_path).one())
o._is_new = False
except sqlalchemy.orm.exc.NoResultFound as e:
if not create:
raise e
a_path = self.filesystem.path(rel_path)
o = File(path=rel_path,
content_hash=Filesystem.file_hash(a_path),
modified=os.path.getmtime(a_path),
process='none'
)
s.add(o)
s.commit()
o._is_new = True
except Exception as e:
return None
return o
def filerec(self, rel_path, create=False):
"""Return a database record for a file."""
import sqlalchemy.orm.exc
s = self.bundle.database.session
if not rel_path:
raise ValueError('Must supply rel_path')
try:
o = (s.query(File).filter(File.path == rel_path).one())
o._is_new = False
except sqlalchemy.orm.exc.NoResultFound as e:
if not create:
raise e
a_path = self.filesystem.path(rel_path)
o = File(path=rel_path,
hash=Filesystem.file_hash(a_path),
modified=os.path.getmtime(a_path),
process='none')
s.add(o)
s.commit()
o._is_new = True
except Exception as e:
return None
return o
def _record_file(self, url, out_file, process = File.PROCESS.DOWNLOADED):
from ambry.util import md5_for_file
from time import time
from sqlalchemy.orm.exc import NoResultFound
from os import stat
# Create a file record of the download
try:
fr = (self.bundle.database.session.query(File)
.filter(File.path == url).filter(File.ref == str(self.bundle.identity.vid))
.filter(File.type_ == File.TYPE.DOWNLOAD).one() )
except NoResultFound:
self.bundle.database.session.rollback()
fr = None
content_hash = md5_for_file(out_file)
stat = os.stat(out_file)
if fr:
if content_hash != fr.hash:
fr.modified = stat.st_mtime
fr.hash = content_hash
fr.process = File.PROCESS.MODIFIED
else:
fr.process = File.PROCESS.UNMODIFIED
else:
fr = File(path=url,
ref=str(self.bundle.identity.vid),
type=File.TYPE.DOWNLOAD,
modified= int(stat.st_mtime),
size = stat.st_size,
hash=content_hash,
process=process,
source_url=url,
data=dict()
)
fr.data['last_download'] = time()
self.bundle.database.session.merge(fr)
self.bundle.database.session.commit()
def get_url(self,source_url, create=False):
'''Return a database record for a file'''
import sqlalchemy.orm.exc
s = self.bundle.database.session
try:
o = (s.query(File).filter(File.source_url==source_url).one())
except sqlalchemy.orm.exc.NoResultFound:
if create:
o = File(source_url=source_url,path=source_url,process='none' )
s.add(o)
s.commit()
else:
return None
o.session = s # Files have SavableMixin
return o
def get_url(self, source_url, create=False):
"""Return a database record for a file."""
import sqlalchemy.orm.exc
s = self.bundle.database.session
try:
o = (s.query(File).filter(File.source_url == source_url).one())
except sqlalchemy.orm.exc.NoResultFound:
if create:
o = File(source_url=source_url,
path=source_url,
process='none')
s.add(o)
s.commit()
else:
return None
o.session = s # Files have SavableMixin
return o
