def add_to_subject_set(subject_set_id,
subject_set_file,
username=None,
password=None):
"""
Import a 1 column file of subject_ids to a subject_set.
Parameters
----------
subject_set_id : str
subject set ID linked to the web interface
subject_set_file : str
one-column file of subject IDs (output of cull_subject_ids)
username, password : str, str
if passed, will add subject set ids to the subject set on the web.
"""
lines = []
with open(subject_set_file) as subject_ids:
lines.append(subject_ids.read().splitlines())
if username is not None:
try:
from panoptes_client import Panoptes, SubjectSet
except ImportError:
print(
'Install https://github.com/zooniverse/panoptes-python-client')
sys.exit(1)
Panoptes.connect(username=username, password=password)
subject_set = SubjectSet.find(subject_set_id)
subject_set.add(np.unique(lines))
return
def subject_set_export(
self,
export_id,
access_token,
):
export = SubjectSetExport.objects.get(pk=export_id)
try:
export.status = SubjectSetExport.RUNNING
export.save()
with SocialPanoptes(bearer_token=access_token) as p:
subject_set = SubjectSet.find(export.subject_set_id)
for subject in subject_set.subjects:
for location in subject.locations:
media_metadata = MediaMetadata.objects.create(
export=export,
subject_id=subject.id,
url=list(location.values())[0],
)
task_result = fetch_media_metadata.delay(media_metadata.id)
media_metadata.celery_task = task_result.id
media_metadata.save()
task_result = update_subject_set_export_status.delay(export_id)
export.celery_task = task_result.id
export.save()
except:
export.status = SubjectSetExport.FAILED
export.save()
raise
def remove_subject_sets(self, workflows_summary):
'''
Removes completed subject sets from workflow by analyzing the workflows summary
returned by get_workflows_summary()
'''
self.log.info(
f"Trying to disable completed Subject Sets from workflows")
if not self._auto_disable_subject_sets:
self.log.info(
f"Configuration prevents from disabling Subject Sets automatically"
)
return
panoptes_client, project = self.get_conn()
for wsum in workflows_summary:
workflow = Workflow.find(wsum['id'])
subject_sets = list()
for ss in wsum['subject_sets']:
if (ss['subjects_count'] != 0) and (ss['retired_count']
== ss['subjects_count']):
subject_set = SubjectSet.find(ss['id'])
subject_sets.append(subject_set)
workflow.remove_subject_sets(subject_sets)
self.log.info(
f"Disabled: {len(subject_sets)} Subject Sets from '{workflow.display_name}'"
)
def make_tutorial_images(imagePaths, ellipseData, projectData):
# Connect to Panoptes
Panoptes.connect(
username=projectData["user_name"], password=projectData["password"]
)
newSubjects = []
for imageId, imagePath in enumerate(imagePaths):
print(f"Adding {imagePath}...")
try:
subjectSet = SubjectSet.find(projectData["subject_set"])
except PanoptesAPIException as e:
print(e)
return
newSubject = Subject()
newSubject.add_location(imagePath)
newSubject.links.project = subjectSet.links.project
newSubject.metadata.update(
make_metadata(
ellipseData.get_group(imageId).reset_index(drop=True), imagePath
)
)
newSubject.save()
newSubjects.append(newSubject)
subjectSet.add(newSubjects)
def download_classifications(subject_set_id, output_file, generate,
generate_timeout):
"""
Downloads a subject-set specific classifications export for the given subject set.
OUTPUT_FILE will be overwritten if it already exists. Set OUTPUT_FILE to -
to output to stdout.
"""
subject_set = SubjectSet.find(subject_set_id)
if generate:
click.echo("Generating new export...", err=True)
export = subject_set.get_export('classifications',
generate=generate,
wait_timeout=generate_timeout)
with click.progressbar(
export.iter_content(chunk_size=1024),
label='Downloading',
length=(int(export.headers.get('content-length')) / 1024 + 1),
file=click.get_text_stream('stderr'),
) as chunks:
for chunk in chunks:
output_file.write(chunk)
def push_new_row_subjects(self, source_subject, target_subject_set_id, row_paths_by_column):
"""
Given image paths for the new column-indexed rows (row_paths_by_column), push new
unclassified row subjects to the appropriate subject set, with metadata references to the
source subject and column.
"""
project = Project.find(settings.PROJECT_ID)
subject_set_unclassified_rows = SubjectSet.find(target_subject_set_id)
new_row_subjects = []
for column_index, row_paths in row_paths_by_column.items():
self._logger.info('Creating %d new row subjects for column index %d for subject %s',
len(row_paths), column_index, source_subject.id)
for row_path in row_paths:
new_subject = Subject()
new_subject.links.project = project
copy_source_metadata_fields = ['book', 'page']
for copy_field in copy_source_metadata_fields:
new_subject.metadata[copy_field] = source_subject.metadata[copy_field]
new_subject.metadata['source_document_subject_id'] = source_subject.id
new_subject.metadata['source_document_column_index'] = column_index
new_subject.add_location(row_path)
new_subject.save()
new_row_subjects.append(new_subject)
subject_set_unclassified_rows.add(new_row_subjects)
def ls(subject_set_id, project_id, workflow_id, quiet):
"""Lists subject set IDs and names"""
if subject_set_id and not project_id and not workflow_id:
subject_set = SubjectSet.find(subject_set_id)
if quiet:
click.echo(subject_set.id)
else:
echo_subject_set(subject_set)
return
args = {}
if project_id:
args['project_id'] = project_id
if workflow_id:
args['workflow_id'] = workflow_id
if subject_set_id:
args['subject_set_id'] = subject_set_id
subject_sets = SubjectSet.where(**args)
if quiet:
click.echo(" ".join([s.id for s in subject_sets]))
else:
for subject_set in subject_sets:
echo_subject_set(subject_set)
def ml_subject_assistant_export_to_microsoft_pt1_get_subjects_data(
export_id,
access_token,
):
print('[Subject Assistant] Exporting to Microsoft 1/4: get Subjects')
export = MLSubjectAssistantExport.objects.get(pk=export_id)
data = [] # Keeps track of all data items that needs to written into a Microsoft-friendly JSON format.
# Retrieve all Subjects from a Subject Set
with SocialPanoptes(bearer_token=access_token) as p:
subject_set = SubjectSet.find(export.subject_set_id)
# Process each Subject
for subject in subject_set.subjects:
# Create a data item for each image URL in the Subject
for frame_id, location in enumerate(subject.locations):
image_url = list(location.values())[0]
subject_information = {
'project_id': str(subject_set.links.project.id),
'subject_set_id': str(export.subject_set_id),
'subject_id': str(subject.id),
'frame_id': str(frame_id)
}
item = []
item.append(image_url)
item.append(json.dumps(subject_information)) # The subject's JSON information is stored as a string. Yes, really.
data.append(item)
return data
def modify(subject_set_id, project_id, display_name):
subject_set = SubjectSet.find(subject_set_id)
if project_id:
subject_set.links.project = project_id
if display_name:
subject_set.display_name = display_name
subject_set.save()
echo_subject_set(subject_set)
def add_to_subject_set(subject_set_id, subject_set_file):
"""Import a 1 column file of subject_ids to a subject_set."""
lines = []
subject_set = SubjectSet.find(subject_set_id)
with open(subject_set_file) as subject_ids:
lines.append(subject_ids.read().splitlines())
return subject_set.add(np.unique(lines))
def _get_subject_set(self, scope, project_id, set_name):
subject_set = None
if not scope.subject_set_id:
subject_set = self._create_subject_set(project_id, set_name)
scope.subject_set_id = subject_set.id
scope.save()
else:
subject_set = SubjectSet.find(scope.subject_set_id)
return subject_set
def remove_subject(self, subject_set_id, subject_list):
"""
:param subject_list:
:return:
"""
subject_set = SubjectSet.find(subject_set_id)
x = subject_set.remove(subject_list)
y = subject_set.save()
return x, y
def modify(subject_set_id, display_name):
"""
Changes the attributes of an existing subject set.
Any attributes which are not specified are left unchanged.
"""
subject_set = SubjectSet.find(subject_set_id)
if display_name:
subject_set.display_name = display_name
subject_set.save()
echo_subject_set(subject_set)
def delete(force, subject_set_ids):
for subject_set_id in subject_set_ids:
subject_set = SubjectSet.find(subject_set_id)
if not force:
click.confirm(
'Delete subject set {} ({})?'.format(
subject_set_id,
subject_set.display_name,
),
abort=True,
)
subject_set.delete()
def remove_subjects(subject_set_id, subject_ids, id_file):
"""
Unlinks subjects from this subject set.
The subjects themselves are not deleted or modified in any way and will
still be present in any other sets they're linked to.
"""
s = SubjectSet.find(subject_set_id)
if id_file:
s.remove([l.strip() for l in id_file.readlines()])
if subject_ids:
s.remove(subject_ids)
def add_subjects(subject_set_id, subject_ids, id_file):
"""
Links existing subjects to this subject set.
This command is useful mainly for adding previously uploaded subjects to
additional subject sets.
See the upload-subjects command to create new subjects in a subject set.
"""
s = SubjectSet.find(subject_set_id)
if id_file:
s.add([l.strip() for l in id_file.readlines()])
if subject_ids:
s.add(subject_ids)
def get_level_structure(self, workflow=None, IDfilter=''):
"""Parameters
----------
workflow : `int`, optional, default None
IDfilter : `str`, optional, default ''
Returns
-------
A dict with keys of workflow IDs and values list
of golden sets associated with that workflow
"""
if hasattr(self, 'level_structure'):
return self.level_structure
level_structure = {}
workflowDictSubjectSets = \
self.get_subject_sets_per_workflow(workflow=workflow)
for iworkflow in workflowDictSubjectSets.keys():
# If it is final workflow level 4 subject sets are also linked
# so need to filter for level 5 subject sets
if int(iworkflow) == 7767:
subjectset_id = [iid for iid in \
workflowDictSubjectSets[iworkflow] \
if iid not in workflowDictSubjectSets['7766']]
else:
subjectset_id = workflowDictSubjectSets[iworkflow]
# Determine Display names of subject set
subjectset_displayname_id = {}
for iSubjectSet in subjectset_id:
tmp2 = SubjectSet.find(iSubjectSet)
displayname = str(tmp2.raw['display_name'])
if IDfilter in displayname:
str_tmp = displayname.split(" '")[0].replace(' ', '_')
subjectset_displayname_id[str_tmp] = \
(iworkflow, iSubjectSet,
[float(iThres)
for iThres
in re.findall("\d+\.\d+", displayname)
]
)
level_structure[iworkflow] = subjectset_displayname_id
self.level_structure = level_structure
return level_structure
def upload_subjects(subject_set_id, manifest_file):
subject_set = SubjectSet.find(subject_set_id)
subject_rows = []
with open(manifest_file) as manifest_f:
file_root = os.path.dirname(manifest_file)
r = csv.reader(manifest_f)
headers = r.next()
for row in r:
metadata = dict(zip(headers, row))
files = []
for col in row:
file_match = re.match(IMAGE_REGEX, col)
file_path = os.path.join(file_root, col)
if file_match and os.path.exists(file_path):
files.append(file_path)
if len(files) == 0:
click.echo('Could not find any files in row:', err=True)
click.echo(','.join(row), err=True)
return -1
subject_rows.append((files, metadata))
created_subjects = []
with click.progressbar(
enumerate(subject_rows),
length=len(subject_rows),
label='Uploading subjects',
) as _subject_rows:
for count, (files, metadata) in _subject_rows:
subject = Subject()
subject.links.project = subject_set.links.project
map(subject.add_location, files)
subject.metadata.update(metadata)
subject.save()
created_subjects.append(subject)
if (count + 1) % LINK_BATCH_SIZE == 0:
subject_set.add(created_subjects)
created_subjects = []
if len(created_subjects) > 0:
subject_set.add(created_subjects)
def get_golden_images(self, workflow=None):
"""Parameters
----------
Returns
-------
A dict with keys of workflow IDs and values list
of golden sets associated with that workflow
"""
workflowGoldenSetImagesDict = {}
workflowGoldenSetDict = self.get_golden_subject_sets()
if workflow:
workflows = workflow
else:
workflows = workflowGoldenSetDict.keys()
for iWorkflow in workflows:
goldenImages = {}
for iGoldenSubjectSet in workflowGoldenSetDict[iWorkflow]:
tmp = SubjectSet.find(iGoldenSubjectSet)
tmpSubjects = tmp.subjects
while True:
try:
nextSubject = tmpSubjects.next()
goldenImages[str(nextSubject.id)] = \
[str(nextSubject.raw['metadata']['subject_id']),\
str(nextSubject.raw['metadata']['#Label']),
int(nextSubject.id)]
except:
break
workflowGoldenSetImagesDict[iWorkflow] = goldenImages
self.workflowGoldenSetImagesDict = workflowGoldenSetImagesDict
return workflowGoldenSetImagesDict
imageio.imwrite('crop.jpeg', image[:,160:1765,:])
# bit rate
#(img) C:/Users/Rdebbout/Downloads/vids_DUL>ffmpeg -i test_out4.mp4 -b 1397520 bit_down2.mp4
# resolution
#(img) C:/Users/Rdebbout/Downloads/vids_DUL>ffmpeg -i test_out4.mp4 -vf scale=960:540 bit_down_scale.mp4
# ffprobe -v quiet -print_format json -show_format -show_streams test_out4.mp4 > op.json
################################################################################
from panoptes_client import SubjectSet, Subject, Project, Panoptes
Panoptes.connect(username='******', password='******')
project = Project.find(id = 5483)
subject_set = SubjectSet.find(17639)
subject = Subject()
subject.links.project = project
subject.add_location({'video/mp4': ('C:/Users/Rdebbout/Downloads/vids_DUL/'
'test_frame_rate/duo_DVR150925_1432_001clip.mp4')})
subject.metadata['site_id'] = 'NCCAGL10-1047'
subject.save()
subject_set.add(subject)
################################################################################
here = 'C:/Users/Rdebbout/Downloads/vids_DUL/test_frame_rate/prepare_ye'
tbl_list = pd.read_csv('CitSci_VideoList_beta.csv')
for f in os.listdir(here):
print f
def make_subject_fits(
full_subject_set_id,
center_workflow_id,
spiral_workflow_id,
bar_workflow_id,
dimensions=[525, 525],
image_location='/Volumes/SD_Extra/manga_images_production/MPL5',
output='MPL5_fits'):
blank_mask = np.zeros(dimensions)
coords = [[x, y] for y in xrange(dimensions[1])
for x in xrange(dimensions[0])]
fullsample = SubjectSet.find(full_subject_set_id)
subjects = fullsample.subjects()
pbar = pb.ProgressBar(widgets=widgets, maxval=subjects.meta['count'])
pbar.start()
idx = 0
for subject in subjects:
subject_metadata = Table(dtype=metadata_dtype)
subject_metadata.add_row(
tuple(subject.raw['metadata'][key]
for key in subject_metadata.dtype.names))
subject_metadata.rename_column('#MANGA_TILEID', 'MANGA_TILEID')
subject_metadata_hdu = fits.table_to_hdu(subject_metadata)
loc = '{0}/{1}_{2}.jpg'.format(
image_location, subject_metadata['MANGAID'][0],
int(subject_metadata['IFUDESIGNSIZE'][0]))
output_name = '{0}/{1}_{2}_{3}.fits'.format(
output, subject_metadata['MANGAID'][0],
int(subject_metadata['IFUDESIGNSIZE'][0]), subject.id)
if os.path.isfile('{0}.gz'.format(output_name)):
# don't process the file if it already exists
idx += 1
pbar.update(idx)
continue
image = plt.imread(loc, format='jpeg')
wcs = define_wcs(subject_metadata['ra'][0], subject_metadata['dec'][0])
wcs_header = wcs.to_header()
orig_image_hdu = fits.PrimaryHDU(data=image, header=wcs_header)
# process data from center(s) and star(s) points
center_classifications = make_classification_table(
'center_points', 'star_points')
all_center = []
all_star = []
for c in Classification.where(scope='project',
workflow_id=center_workflow_id,
subject_id=subject.id):
record_base_classification(c, center_classifications)
center_points = []
star_points = []
points = c.raw['annotations'][0]['value']
for p in points:
if ('x' in p) and ('y' in p):
if p['tool'] == 0:
# somehow the workflow_id got messed up for some classifications
# so a try statement is needed
loc = [p['x'], p['y']]
center_points.append(loc)
all_center.append(loc)
elif p['tool'] == 1:
loc = [p['x'], p['y']]
star_points.append(loc)
all_star.append(loc)
center_classifications['center_points'].append(
json.dumps(center_points))
center_classifications['star_points'].append(
json.dumps(star_points))
center_star_table_hdu = fits.table_to_hdu(
Table(center_classifications))
# cluster points and make image masks
if len(all_center):
center_mask, center_table_hdu = cluster(np.array(all_center),
dimensions, coords, wcs)
else:
center_mask = blank_mask
center_table_hdu = fits.table_to_hdu(Table(make_cluster_table()))
center_hdu = fits.ImageHDU(data=center_mask, header=wcs_header)
if len(all_star):
star_mask, star_table_hdu = cluster(np.array(all_star), dimensions,
coords, wcs)
else:
star_mask = blank_mask
star_table_hdu = fits.table_to_hdu(Table(make_cluster_table()))
star_hdu = fits.ImageHDU(data=star_mask, header=wcs_header)
# spiral arms
spiral_table_hdu, spiral_hdu = mask_process(spiral_workflow_id,
subject.id, 'spiral_paths',
wcs_header, dimensions,
coords)
# bars
bar_table_hdu, bar_hdu = mask_process(bar_workflow_id, subject.id,
'bar_paths', wcs_header,
dimensions, coords)
# make fits file
hdu_list = fits.HDUList([
orig_image_hdu, center_hdu, star_hdu, spiral_hdu, bar_hdu,
subject_metadata_hdu, center_table_hdu, star_table_hdu,
center_star_table_hdu, spiral_table_hdu, bar_table_hdu
])
hdu_list.writeto(output_name)
# compress the fits file
call(['gzip', output_name])
# update progressbar
idx += 1
pbar.update(idx)
pbar.finish()
def ls(subject_set_id):
echo_subject_set(SubjectSet.find(subject_set_id))
def upload_subjects(
subject_set_id,
manifest_files,
allow_missing,
remote_location,
mime_type,
file_column,
):
"""
Uploads subjects from each of the given MANIFEST_FILES.
Example with only local files:
$ panoptes subject-set upload-subjects 4667 manifest.csv
Local filenames will be automatically detected in the manifest and
uploaded, or filename columns can be specified with --file-column.
If you are hosting your media yourself, you can put the URLs in the
manifest and specify the column number(s):
$ panoptes subject-set upload-subjects -r 1 4667 manifest.csv
$ panoptes subject-set upload-subjects -r 1 -r 2 4667 manifest.csv
Any local files will still be detected and uploaded.
"""
if (
len(manifest_files) > 1
and any(map(lambda m: m.endswith('.yaml'), manifest_files))
):
click.echo(
'Error: YAML manifests must be processed one at a time.',
err=True,
)
return -1
elif manifest_files[0].endswith('.yaml'):
with open(manifest_files[0], 'r') as yaml_manifest:
upload_state = yaml.load(yaml_manifest, Loader=yaml.FullLoader)
if upload_state['state_version'] > CURRENT_STATE_VERSION:
click.echo(
'Error: {} was generated by a newer version of the Panoptes '
'CLI and is not compatible with this version.'.format(
manifest_files[0],
),
err=True,
)
return -1
if upload_state['subject_set_id'] != subject_set_id:
click.echo(
'Warning: You specified subject set {} but this YAML '
'manifest is for subject set {}.'.format(
subject_set_id,
upload_state['subject_set_id'],
),
err=True,
)
click.confirm(
'Upload {} to subject set {} ({})?'.format(
manifest_files[0],
subject_set_id,
SubjectSet.find(subject_set_id).display_name,
),
abort=True
)
upload_state['subject_set_id'] = subject_set_id
resumed_upload = True
else:
upload_state = {
'state_version': CURRENT_STATE_VERSION,
'subject_set_id': subject_set_id,
'manifest_files': manifest_files,
'allow_missing': allow_missing,
'remote_location': remote_location,
'mime_type': mime_type,
'file_column': file_column,
'waiting_to_upload': [],
'waiting_to_link': {},
}
resumed_upload = False
remote_location_count = len(upload_state['remote_location'])
mime_type_count = len(upload_state['mime_type'])
if remote_location_count > 1 and mime_type_count == 1:
upload_state['mime_type'] = (
upload_state['mime_type'] * remote_location_count
)
elif remote_location_count > 0 and mime_type_count != remote_location_count:
click.echo(
'Error: The number of MIME types given must be either 1 or equal '
'to the number of remote locations.',
err=True,
)
return -1
def validate_file(file_path):
if not os.path.isfile(file_path):
click.echo(
'Error: File "{}" could not be found.'.format(
file_path,
),
err=True,
)
return False
file_size = os.path.getsize(file_path)
if file_size == 0:
click.echo(
'Error: File "{}" is empty.'.format(
file_path,
),
err=True,
)
return False
elif file_size > MAX_UPLOAD_FILE_SIZE:
click.echo(
'Error: File "{}" is {}, larger than the maximum {}.'.format(
file_path,
humanize.naturalsize(file_size),
humanize.naturalsize(MAX_UPLOAD_FILE_SIZE),
),
err=True,
)
return False
return True
subject_set = SubjectSet.find(upload_state['subject_set_id'])
if not resumed_upload:
subject_rows = []
for manifest_file in upload_state['manifest_files']:
with open(manifest_file, 'U') as manifest_f:
file_root = os.path.dirname(manifest_file)
r = csv.reader(manifest_f, skipinitialspace=True)
headers = next(r)
for row in r:
metadata = dict(zip(headers, row))
files = []
if not upload_state['file_column']:
upload_state['file_column'] = []
for field_number, col in enumerate(row, start=1):
file_path = os.path.join(file_root, col)
if os.path.exists(file_path):
upload_state['file_column'].append(
field_number,
)
if not validate_file(file_path):
return -1
files.append(file_path)
else:
for field_number in upload_state['file_column']:
file_path = os.path.join(
file_root,
row[field_number - 1]
)
if not validate_file(file_path):
return -1
files.append(file_path)
for field_number, _mime_type in zip(
upload_state['remote_location'],
upload_state['mime_type'],
):
files.append({_mime_type: row[field_number - 1]})
if len(files) == 0:
click.echo(
'Could not find any files in row:',
err=True,
)
click.echo(','.join(row), err=True)
if not upload_state['allow_missing']:
return -1
else:
continue
subject_rows.append((files, metadata))
if not subject_rows:
click.echo(
'File {} did not contain any rows.'.format(
manifest_file,
),
err=True,
)
return -1
subject_rows = list(enumerate(subject_rows))
upload_state['waiting_to_upload'] = copy.deepcopy(subject_rows)
else:
for subject_id, subject_row in upload_state['waiting_to_link'].items():
try:
subject = Subject.find(subject_id)
except PanoptesAPIException:
upload_state['waiting_to_upload'].append(subject_row)
del upload_state['waiting_to_link'][subject_id]
subject_rows = copy.deepcopy(upload_state['waiting_to_upload'])
pending_subjects = []
def move_created(limit):
while len(pending_subjects) > limit:
for subject, subject_row in pending_subjects:
if subject.async_save_result:
pending_subjects.remove((subject, subject_row))
upload_state['waiting_to_upload'].remove(subject_row)
upload_state['waiting_to_link'][subject.id] = subject_row
time.sleep(0.5)
def link_subjects(limit):
if len(upload_state['waiting_to_link']) > limit:
subject_set.add(list(upload_state['waiting_to_link'].keys()))
upload_state['waiting_to_link'].clear()
with click.progressbar(
subject_rows,
length=len(subject_rows),
label='Uploading subjects',
) as _subject_rows:
try:
with Subject.async_saves():
for subject_row in _subject_rows:
count, (files, metadata) = subject_row
subject = Subject()
subject.links.project = subject_set.links.project
for media_file in files:
subject.add_location(media_file)
subject.metadata.update(metadata)
subject.save()
pending_subjects.append((subject, subject_row))
move_created(MAX_PENDING_SUBJECTS)
link_subjects(LINK_BATCH_SIZE)
move_created(0)
link_subjects(0)
finally:
if (
len(pending_subjects) > 0
or len(upload_state['waiting_to_link']) > 0
):
click.echo('Error: Upload failed.', err=True)
if click.confirm(
'Would you like to save the upload state to resume the '
'upload later?',
default=True,
):
while True:
state_file_name = 'panoptes-upload-{}.yaml'.format(
subject_set_id,
)
state_file_name = click.prompt(
'Enter filename to save to',
default=state_file_name,
)
if not state_file_name.endswith('.yaml'):
click.echo(
'Error: File name must end in ".yaml".',
err=True,
)
if click.confirm(
'Save to {}.yaml?'.format(state_file_name),
default=True,
):
state_file_name += '.yaml'
else:
continue
if not is_valid_filename(state_file_name):
click.echo(
'Error: {} is not a valid file name'.format(
state_file_name,
),
err=True,
)
sanitized_filename = sanitize_filename(
state_file_name,
)
if click.confirm(
'Save to {}?'.format(
sanitized_filename,
),
default=True,
):
state_file_name = sanitized_filename
else:
continue
if os.path.exists(state_file_name):
if not click.confirm(
'File {} already exists. Overwrite?'.format(
state_file_name,
),
default=False,
):
continue
break
with open(state_file_name, 'w') as state_file:
yaml.dump(upload_state, state_file)
subject.links.project = project
subject.add_location(file)
# You can set whatever metadata you want, or none at all
subject.metadata['filename'] = os.path.basename(file)
#TODO subject.metadata['file_start'] =
#TODO subject.metadata['sample_rate'] = 5512
subject.metadata['fft'] = fft
subject.metadata['overlap'] = overlap
subject.metadata['color_min'] = color_min
subject.metadata['color_max'] = color_max
#TODO subject.metadata['width'] =
#TODO subject.metadata['height'] =
subject.save()
subjects.append(subject)
os.rename(file,dest+os.path.basename(file)) #move file to uploaded directory
#Create a new subject set or append the subjects to an existing one
for subject_set in project.links.subject_sets:
if str(subject_set.display_name) == subject_set_display_name:
subject_set_id = subject_set.id
subject_set = SubjectSet.find(subject_set_id)
break
else:
#subject_set = SubjectSet()
#subject_set.links.project = project
#subject_set.display_name = subject_set_display_name
#subject_set.save()
raise Exception('Subject set does not exist')
subject_set.add(subjects) # SubjectSet.add_subjects() can take a list of Subjects, or just one.
print("--- %s seconds ---" % (time.time() - start_time))
args = parser.parse_args()
username = input(f"Enter Zooiverse username:"******"Enter Zooniverse password for {username}: ")
if args.password is None
else args.password
)
projectData = {
"subject_set": args.subjectSetId,
"user_name": username,
"password": password,
}
imagePaths = pd.read_csv(args.imagePaths, header=None)[0]
ellipseData = pd.read_csv(args.ellipseData).groupby(by="image_id")
try:
print(
"Adding subjects to {} {}".format(
SubjectSet.find(projectData["subject_set"]).display_name,
projectData["subject_set"],
)
)
except PanoptesAPIException as e:
print(e)
exit(1)
make_tutorial_images(imagePaths, ellipseData, projectData)
and add the datetime to the subject metadata. It requires the project owner credentials
to be set up as OS environmental variables, and an appropriate project slug modified
on line 11. depending on the camera used to take the original subject image the exif
code may be different than that in the code and may need to be modified"""
import os
from PIL import Image, ExifTags
import panoptes_client
from panoptes_client import SubjectSet, Project, Panoptes
import requests
Panoptes.connect(username=os.environ['User_name'], password=os.environ['Password'])
project = Project.find(slug='pmason\fossiltrainer')
while True:
set_id = input('Entry subject set id to update:' + '\n')
try:
subject_set = SubjectSet.find(set_id)
count_subjects = 0
subject_list = []
for subject in subject_set.subjects:
count_subjects += 1
if subject.metadata['DateTime'] == '':
try:
img = Image.open(requests.get(subject.locations[0]['image/jpeg'], stream=True).raw)
exif_dict = img._getexif()
date_time = exif_dict[306]
except (IOError, KeyError):
print('Acquiring exif data for ', subject.id, ' failed')
continue
subject.metadata['DateTime'] = date_time
print(subject.id, subject.metadata['DateTime'], ' fixed')
subject.save()
# 1) Edit input parameters below.
# 2) Run from command line: `python edit_metadata.py`
from panoptes_client import Panoptes, SubjectSet
###################################
# Input Parameters
puser = '******'
ppswd = 'PASSWORD'
subject_set_ids = [77172]
new_metadata = {
"Attribution":
"Photos are open source under a CC BY 3.0 license and were contributed to the project via CitSci.org."
}
###################################
Panoptes.connect(username=puser, password=ppswd)
for ssid in subject_set_ids:
subject_set = SubjectSet.find(ssid)
print('Editing metadata for Subject Set #{0}: {1}'.format(
subject_set.raw['id'], subject_set.raw['display_name']))
for subject in subject_set.subjects:
for key, value in new_metadata.items():
subject.metadata[key] = value
subject.save()
def info(subject_set_id):
subject_set = SubjectSet.find(subject_set_id)
click.echo(yaml.dump(subject_set.raw))
def _get_subject_set(self, subject_set_id):
if subject_set_id not in self._subject_sets:
self._subject_sets[subject_set_id] = SubjectSet.find(
subject_set_id)
return self._subject_sets[subject_set_id]
4) Remove the second set of RA and DEC columns, and name the first ones 'RA' and 'DEC'
5) Save this file as a csv
6) This only works in python 2 because it uses panoptes_client
'''
import numpy as np
from panoptes_client import SubjectSet, Subject, Project, Panoptes
import os
import progressbar as pb
myusername = os.environ['PANOPTES_USERNAME']
mypassword = os.environ['PANOPTES_PASSWORD']
Panoptes.connect(username= myusername, password=mypassword)
project = Project.find(id='73')
fullsample = SubjectSet.find(5326)
spirals = SubjectSet.find(5324)
bars = SubjectSet.find(5325)
progress = pb.ProgressBar(widgets=[pb.Bar(), pb.ETA()])
data = np.genfromtxt('../GZ3D/MatchedData.csv', delimiter = ',', names=True,
dtype=[('DEC', float), ('IAUNAME', '|S30'),('IFUTARGETSIZE',int),
('MANGAID', '|S10'),('MANGA_TILEID',int),('NSAID', int),
('PETROTH50',float),('RA',float),('SERSIC_TH50',float),
('Z',float),('specobjid', int),('dr8objid', int),
('dr7objid', int),('t01_smooth_or_features_a02_features_or_disk_weighted_fraction', float),
('t02_edgeon_a05_no_weighted_fraction', float),
('t03_bar_a06_bar_weighted_fraction', float),
('t04_spiral_a08_spiral_weighted_fraction', float)])
counter = 0
nancounter = 0
spiralcount = 0
#!/usr/bin/env python3
import sys
sys.path.insert(0, "..")
import pdb
from panoptes_client import Panoptes, SubjectSet
from lib import settings
Panoptes.connect(username=settings.PANOPTES_USERNAME,
password=settings.PANOPTES_PASSWORD)
subject_set = SubjectSet.find(14804)
print(' '.join([s.id for s in subject_set.subjects]))
