class KerberosKeys(models.Model):
name = models.CharField(max_length=100)
username = models.CharField(max_length=100)
file = BlobField()
class Meta:
db_table = 'kerberos_keys'
class ExecutionJobs(models.Model):
execution_id = models.IntegerField()
node_id = models.PositiveSmallIntegerField()
project_id = models.PositiveIntegerField()
project_version = models.PositiveIntegerField()
flow_id = models.CharField(max_length=255)
job_id = models.CharField(max_length=255)
status = models.CharField(max_length=32)
start_time = models.DateTimeField(null=True)
end_time = models.DateTimeField(null=True)
update_time = models.DateTimeField(auto_now_add=True)
input_params = BlobField()
output_params = BlobField()
class Meta:
db_table = 'execution_jobs'
unique_together = ('execution_id', 'job_id')
class ProjectFiles(models.Model):
project_id = models.IntegerField(db_index=True)
project_version = models.IntegerField(db_index=True)
chunk = models.IntegerField()
file = BlobField()
class Meta:
db_table = 'project_files'
class Trigger(models.Model):
status = models.CharField(max_length=16, default="READY")
encoding_type = models.CharField(max_length=16, default="PLAIN")
data = BlobField()
create_time = models.DateTimeField()
update_time = models.DateTimeField(auto_now_add=True)
class Meta:
db_table = 'trigger'
class FlowLogs(models.Model):
execution_id = models.AutoField(primary_key=True)
node_id = models.PositiveSmallIntegerField()
encoding_type = models.CharField(max_length=16)
start_byte = models.PositiveIntegerField()
end_byte = models.PositiveIntegerField()
upload_time = models.DateTimeField(auto_now_add=True)
log = BlobField()
class Meta:
db_table = 'flow_logs'
class JobLogs(models.Model):
execution_id = models.IntegerField()
node_id = models.PositiveSmallIntegerField()
job_id = models.CharField(max_length=255)
encoding_type = models.CharField(max_length=16)
start_byte = models.PositiveIntegerField()
end_byte = models.PositiveIntegerField()
upload_time = models.DateTimeField(auto_now_add=True)
log = BlobField()
class Meta:
db_table = 'job_logs'
unique_together = ('execution_id', 'job_id')
class Schedule(models.Model):
node_id = models.IntegerField()
schedule_user = models.CharField(max_length=64)
project_id = models.IntegerField(max_length=10)
project_version = models.IntegerField(max_length=10)
flow_id = models.CharField(max_length=128)
flow_data = BlobField()
encoding_type = models.CharField(max_length=16, default='PLAIN')
first_check_time = models.CharField(max_length=32)
period = models.CharField(max_length=32)
timezone = models.CharField(max_length=32)
status = models.CharField(max_length=16, default='READY')
create_time = models.DateTimeField()
update_time = models.DateTimeField(auto_now_add=True)
class Meta:
db_table = 'schedule'
unique_together = ('id', 'project_id', 'project_version', 'flow_id')
class ExecutionFlows(models.Model):
execution_id = models.AutoField(primary_key=True)
schedule_id = models.IntegerField(default=0)
node_id = models.PositiveSmallIntegerField()
project_id = models.PositiveIntegerField(db_index=True)
project_version = models.PositiveIntegerField()
flow_id = models.CharField(max_length=255)
flow_data = BlobField()
encoding_type = models.CharField(max_length=16)
status = models.CharField(max_length=32)
submit_user = models.CharField(max_length=255)
submit_time = models.DateTimeField(auto_now_add=True)
start_time = models.DateTimeField(null=True)
end_time = models.DateTimeField(null=True)
update_time = models.DateTimeField(null=True)
class Meta:
db_table = 'execution_flows'
@staticmethod
def create(self, flow):
'''@type: flows.Flow'''
ExecutionFlows.objects.create(flow)
class ModuleVersions(models.Model):
module_id = models.IntegerField(max_length=10, db_index=True)
type = models.CharField(max_length=100)
comment = models.CharField(max_length=200, default='')
file = BlobField()
hdfs_file_id = models.IntegerField(max_length=10, default=0)
file_name = models.CharField(max_length=100, default='')
git_options = models.CharField(max_length=500, null=True)
options = models.CharField(max_length=10000, default='') # custom extended parameters
created_by = models.CharField(max_length=100)
created_time = models.DateTimeField(auto_now_add=True)
refer_count = models.IntegerField(max_length=10, default=0)
status = models.SmallIntegerField(max_length=3, default=0)
@classmethod
def create(cls, module_id, comment, version_type, file_name, git_options, keep_old_file, options, created_by):
file_content = "" # used by module save mode == DB
hdfs_file_id = 0 # used by module save mode == HDFS
if keep_old_file:
module_version_latest = ModuleVersions.objects.filter(module_id=module_id, status=0).order_by("-id")[:1].get()
if settings.REPOSITORY_SAVE_MODE == settings.REPOSITORY_SAVE_MODE_HDFS:
hdfs_file_id = module_version_latest.hdfs_file_id
else:
file_content = module_version_latest.file
elif has_file(file_name):
if settings.REPOSITORY_SAVE_MODE == settings.REPOSITORY_SAVE_MODE_HDFS:
uploaded_file_path = settings.MEDIA_PATH + file_name
checksum = checksum_file(open(uploaded_file_path))
webhdfs = WebHdfs()
remote_file_path = settings.MODULE_HDFS_ROOT + file_name + "_" + checksum
logger.info("hdfs copy from local %s to remote %s", uploaded_file_path, remote_file_path)
webhdfs.copyFromLocal(uploaded_file_path, remote_file_path)
module_file = ModuleHdfsFiles(name=file_name,
checksum=checksum,
path=remote_file_path)
module_file.save()
hdfs_file_id = module_file.id
else:
file_raw = get_uploaded_file_content(file_name)
file_content = base64.b64encode("".join(file_raw))
else:
raise ObjectDoesNotExist
logger.info('create version --')
module_version = cls(module_id=module_id,
comment=comment,
type=version_type,
file=file_content,
hdfs_file_id=hdfs_file_id,
file_name=file_name,
git_options=git_options,
options=options,
created_by=created_by,
created_time=timezone.now())
module_version.save()
logger.info('create version ++')
logger.debug("module version is created, id %s, type %s, file_name %s, git_options %s, options %s",
module_id, version_type, file_name, git_options, options)
return module_version
class Meta:
db_table = 'module_versions'