def handle(self, username, base_isa_dir, **options):
try:
self._username = username
except:
raise CommandError(self.help)
"""get a list of all the isatab files in base_isa_dir"""
isatab_dict = dict()
for dirname, dirnames, filenames in os.walk(base_isa_dir):
for filename in filenames:
isatab_dict[filename] = [os.path.join(dirname, filename)]
"""If isatab_dict is empty, then base_isa_dir is a file, not a directory"""
if not isatab_dict:
isatab_dict[base_isa_dir] = [base_isa_dir]
"""get a list of all the isatab files in base_pre_isa_dir"""
pre_isatab_files = 0
try:
for dirname, dirnames, filenames in os.walk(options['base_pre_isa_dir']):
for filename in filenames:
"""associate pre-isatab file with isatab file"""
for key in isatab_dict:
if re.search(r'%s$' % key, filename):
file = os.path.join(dirname, filename)
isatab_dict[key].append(file)
pre_isatab_files += 1
break
except:
pass
"""
If base_pre_isa_dir is defined but pre_isatab_files is 0,
then base_pre_isa_dir is a pre-ISA-Tab archive, not a directory
"""
if options['base_pre_isa_dir'] and not pre_isatab_files:
isatab_dict[base_isa_dir].append(options['base_pre_isa_dir'])
s_tasks = list()
"""add subtasks to list"""
for k, v_list in isatab_dict.items():
isa_file = v_list.pop(0)
try:
pre_file = v_list.pop(0)
except:
pre_file = None
sub_task = parse_isatab.subtask(args=(
self._username,
options['public'],
isa_file,
self._additional_raw_data_file_extension,
isa_file, pre_file,
options['file_base_path']
)
)
s_tasks.append(sub_task)
job = TaskSet(tasks=s_tasks)
result = job.apply_async()
for i in result.iterate():
try:
ds = DataSet.objects.get(uuid=i)
inv = ds.get_investigation()
print "Successfully parsed %s into DataSet" % inv.get_identifier(),
print "with UUID %s" % i
sys.stdout.flush()
except:
print "Unsuccessful parse and DataSet Creation of %s." % i
sys.stdout.flush()
def handle(self, username, base_isa_dir, **options):
try:
self._username = username
except:
raise CommandError(self.help)
# Get a list of all the isatab files in base_isa_dir
isatab_dict = OrderedDict()
for root, dirs, filenames in os.walk(base_isa_dir):
for filename in filenames:
_, extension = os.path.splitext(filename)
if extension.lower() == '.zip':
isatab_dict[filename] = [os.path.join(root, filename)]
isatab_dict = OrderedDict(sorted(isatab_dict.iteritems()))
# If isatab_dict is empty, then base_isa_dir is a file, not a directory
if not isatab_dict:
isatab_dict[base_isa_dir] = [base_isa_dir]
# Get a list of all the isatab files in base_pre_isa_dir
pre_isatab_files = 0
try:
for root, dirs, filenames in os.walk(options['base_pre_isa_dir']):
for filename in filenames:
# Associate pre-isatab file with isatab file
for key in isatab_dict:
if re.search(r'%s$' % key, filename):
file = os.path.join(root, filename)
isatab_dict[key].append(file)
pre_isatab_files += 1
break
except:
pass
"""
If base_pre_isa_dir is defined but pre_isatab_files is 0,
then base_pre_isa_dir is a pre-ISA-Tab archive, not a directory
"""
if options['base_pre_isa_dir'] and not pre_isatab_files:
isatab_dict[base_isa_dir].append(options['base_pre_isa_dir'])
s_tasks = list()
# Add subtasks to list
for k, v_list in isatab_dict.items():
isa_file = v_list.pop(0)
try:
pre_file = v_list.pop(0)
except:
pre_file = None
sub_task = parse_isatab.subtask(
args=(self._username, options['public'], isa_file,
self._additional_raw_data_file_extension, isa_file,
pre_file, options['file_base_path']))
s_tasks.append(sub_task)
job = TaskSet(tasks=s_tasks)
result = job.apply_async()
task_num = 1
total = len(isatab_dict)
for (uuid, filename, skipped) in result.iterate():
try:
if not skipped:
logger.info(
"%s / %s: Successfully parsed %s into "
"DataSet with UUID %s", task_num, total, filename,
uuid)
else:
logger.info(
"%s / %s: Skipped %s as it has been "
"successfully parsed already. UUID %s", task_num,
total, filename, uuid)
task_num += 1
sys.stdout.flush()
except:
logger.info("%s / %s: Unsuccessful parsed %s", task_num, total,
filename)
task_num += 1
sys.stdout.flush()
def handle(self, username, base_isa_dir, **options):
try:
self._username = username
except:
raise CommandError(self.help)
# Get a list of all the isatab files in base_isa_dir
isatab_dict = OrderedDict()
for root, dirs, filenames in os.walk(base_isa_dir):
for filename in filenames:
_, extension = os.path.splitext(filename)
if extension.lower() == '.zip':
isatab_dict[filename] = [os.path.join(root, filename)]
isatab_dict = OrderedDict(sorted(isatab_dict.iteritems()))
# If isatab_dict is empty, then base_isa_dir is a file, not a directory
if not isatab_dict:
isatab_dict[base_isa_dir] = [base_isa_dir]
# Get a list of all the isatab files in base_pre_isa_dir
pre_isatab_files = 0
try:
for root, dirs, filenames in os.walk(options['base_pre_isa_dir']):
for filename in filenames:
# Associate pre-isatab file with isatab file
for key in isatab_dict:
if re.search(r'%s$' % key, filename):
file = os.path.join(root, filename)
isatab_dict[key].append(file)
pre_isatab_files += 1
break
except:
pass
"""
If base_pre_isa_dir is defined but pre_isatab_files is 0,
then base_pre_isa_dir is a pre-ISA-Tab archive, not a directory
"""
if options['base_pre_isa_dir'] and not pre_isatab_files:
isatab_dict[base_isa_dir].append(options['base_pre_isa_dir'])
s_tasks = list()
# Add subtasks to list
for k, v_list in isatab_dict.items():
isa_file = v_list.pop(0)
try:
pre_file = v_list.pop(0)
except:
pre_file = None
sub_task = parse_isatab.subtask(
args=(
self._username,
options['public'],
isa_file,
self._additional_raw_data_file_extension,
isa_file, pre_file,
options['file_base_path']
)
)
s_tasks.append(sub_task)
job = TaskSet(tasks=s_tasks)
result = job.apply_async()
task_num = 1
total = len(isatab_dict)
for (uuid, filename, skipped) in result.iterate():
try:
if not skipped:
logger.info(
"%s / %s: Successfully parsed %s into "
"DataSet with UUID %s",
task_num, total, filename, uuid)
else:
logger.info(
"%s / %s: Skipped %s as it has been "
"successfully parsed already. UUID %s",
task_num, total, filename, uuid)
task_num += 1
sys.stdout.flush()
except:
logger.info("%s / %s: Unsuccessful parsed %s",
task_num, total, filename)
task_num += 1
sys.stdout.flush()
