def __init__(self, file, name, path, scan):
super(FileWeb, self).__init__()
self.external_id = UUID.generate()
self.file = file
self.name = name
self.path = path
self.scan = scan
def __init__(self, file, name, path, scan):
super(FileWeb, self).__init__()
self.external_id = UUID.generate()
self.file = file
self.name = name
self.path = path
self.scan = scan
def __init__(self, date, ip):
super(Scan, self).__init__()
self.external_id = UUID.generate()
self.date = date
self.ip = ip
self.force = False
self.mimetype_filtering = False
self.resubmit_files = False
def __init__(self, date, ip):
super(Scan, self).__init__()
self.external_id = UUID.generate()
self.date = date
self.ip = ip
self.force = False
self.mimetype_filtering = False
self.resubmit_files = False
def upgrade():
bind = op.get_bind()
session = scoped_session(sessionmaker(autocommit=False, autoflush=False,
bind=bind))
op.add_column('irma_file', sa.Column('mimetype',
sa.String(),
nullable=True))
op.add_column('irma_fileWeb', sa.Column('external_id',
sa.String(length=36),
nullable=True))
op.add_column('irma_fileWeb', sa.Column('id_parent',
sa.Integer(),
nullable=True))
op.add_column('irma_fileWeb', sa.Column('path',
sa.String(length=255),
nullable=True))
# Create external_id as new uuid
for fileweb in session.query(FileWeb).all():
if fileweb.external_id is None:
fileweb.external_id = UUID.generate()
session.commit()
# Now that all data are fixed set column to non nullable
op.alter_column('irma_fileWeb', 'external_id', nullable=False)
op.create_index(op.f('ix_irma_fileWeb_external_id'),
'irma_fileWeb',
['external_id'],
unique=False)
op.drop_constraint(u'irma_fileWeb_id_scan_scan_file_idx_key',
'irma_fileWeb',
type_='unique')
op.create_unique_constraint(None,
'irma_fileWeb',
['external_id'])
op.create_foreign_key(None,
'irma_fileWeb',
'irma_file',
['id_parent'],
['id'])
op.drop_column('irma_fileWeb', 'scan_file_idx')
op.add_column('irma_scan', sa.Column('force',
sa.Boolean(),
nullable=True))
op.add_column('irma_scan', sa.Column('mimetype_filtering',
sa.Boolean(),
nullable=True))
op.add_column('irma_scan', sa.Column('probelist',
sa.String(),
nullable=True))
op.add_column('irma_scan', sa.Column('resubmit_files',
sa.Boolean(),
nullable=True))
op.add_column('irma_tag', sa.Column('text',
sa.String(),
nullable=False))
op.drop_column('irma_tag', 'name')
def setUp(self):
self.user_name = "test_user"
self.user_rmqvhost = "test_vhost"
self.user_ftpuser = "******"
self.user_quota = 100
self.scanid = UUID.generate()
try:
self.userid = user_ctrl.get_userid(self.user_rmqvhost)
except IrmaDatabaseResultNotFound:
# user does not exist create one
with session_transaction() as session:
user = User(self.user_name,
self.user_rmqvhost,
self.user_ftpuser,
self.user_quota)
user.save(session)
session.commit()
self.userid = user.id
def test005_validate_id(self):
uuid = UUID.generate()
self.assertIsNone(module.validate_id(uuid))
def validate_id(ext_id):
""" check external_id format - should be a str(ObjectId)"""
if not UUID.validate(ext_id):
raise ValueError("Malformed id")
def __init__(self, file, name):
self.external_id = UUID.generate()
self.file = file
self.name = name
def __init__(self, scanid, filename, probename):
self.task_id = UUID.generate()
self.scan_id = scanid
self.filename = filename
self.probename = probename
def validate_scanid(scanid):
""" check scanid format - should be a str(ObjectId)"""
if not UUID.validate(scanid):
raise ValueError("Malformed Scanid")
def scan(frontend_scanid, scan_request):
try:
log.info("%s", frontend_scanid)
# TODO user should be identified by RMQ vhost
# read vhost from config as workaround
rmqvhost = config.get_frontend_rmqvhost()
user_id = user_ctrl.get_userid(rmqvhost)
ftpuser = user_ctrl.get_ftpuser(user_id)
# create an initial entry to track future errors
# tell frontend that frontend_scanid is now known to brain
# progress available
scan_id = scan_ctrl.new(frontend_scanid, user_id, len(scan_request))
# send a scan launched event to frontend
(remaining, quota) = user_ctrl.get_quota(user_id)
if remaining is not None:
log.info("%s quota remaining {0}/{1}",
frontend_scanid,
remaining,
quota)
else:
log.info("%s unlimited quota",
frontend_scanid)
if remaining <= 0:
scan_ctrl.error(scan_id, IrmaScanStatus.error)
available_probelist = celery_probe.get_probelist()
for (filename, probe_list) in scan_request.items():
if probe_list is None:
scan_ctrl.error(scan_id, IrmaScanStatus.error_probe_missing)
log.info("%s Empty probe list", frontend_scanid)
return IrmaTaskReturn.error("empty probe list on brain")
# first check probe_list
for p in probe_list:
# check if probe exists
if p not in available_probelist:
scan_ctrl.error(scan_id,
IrmaScanStatus.error_probe_missing)
msg = "Unknown probe {0}".format(p)
log.info("%s Unknown probe %s",
frontend_scanid,
p)
return IrmaTaskReturn.error(msg)
# Now, create one subtask per file to
# scan per probe according to quota
for probe in probe_list:
if remaining is not None:
if remaining <= 0:
break
else:
remaining -= 1
task_id = UUID.generate()
job_id = job_ctrl.new(scan_id, filename, probe, task_id)
celery_probe.job_launch(ftpuser,
frontend_scanid,
filename,
probe,
job_id,
task_id)
scan_ctrl.launched(scan_id)
celery_frontend.scan_launched(frontend_scanid)
log.info(
"%s %d files receives / %d active probe /" +
" %d probe used / %d jobs launched",
frontend_scanid,
len(scan_request),
len(available_probelist),
len(probe_list),
scan_ctrl.get_nbjobs(scan_id))
except:
log.info("exception", exc_info=True)
raise
def test005_validate_id(self):
uuid = UUID.generate()
self.assertIsNone(module.validate_id(uuid))
def __init__(self, scanid, filehash, probename):
self.scan_id = scanid
self.task_id = UUID.generate()
self.filehash = filehash
self.probename = probename