def add_to_subject_set(subject_set_id, subject_set_file, username=None, password=None): """ Import a 1 column file of subject_ids to a subject_set. Parameters ---------- subject_set_id : str subject set ID linked to the web interface subject_set_file : str one-column file of subject IDs (output of cull_subject_ids) username, password : str, str if passed, will add subject set ids to the subject set on the web. """ lines = [] with open(subject_set_file) as subject_ids: lines.append(subject_ids.read().splitlines()) if username is not None: try: from panoptes_client import Panoptes, SubjectSet except ImportError: print( 'Install https://github.com/zooniverse/panoptes-python-client') sys.exit(1) Panoptes.connect(username=username, password=password) subject_set = SubjectSet.find(subject_set_id) subject_set.add(np.unique(lines)) return
def subject_set_export( self, export_id, access_token, ): export = SubjectSetExport.objects.get(pk=export_id) try: export.status = SubjectSetExport.RUNNING export.save() with SocialPanoptes(bearer_token=access_token) as p: subject_set = SubjectSet.find(export.subject_set_id) for subject in subject_set.subjects: for location in subject.locations: media_metadata = MediaMetadata.objects.create( export=export, subject_id=subject.id, url=list(location.values())[0], ) task_result = fetch_media_metadata.delay(media_metadata.id) media_metadata.celery_task = task_result.id media_metadata.save() task_result = update_subject_set_export_status.delay(export_id) export.celery_task = task_result.id export.save() except: export.status = SubjectSetExport.FAILED export.save() raise
def remove_subject_sets(self, workflows_summary): ''' Removes completed subject sets from workflow by analyzing the workflows summary returned by get_workflows_summary() ''' self.log.info( f"Trying to disable completed Subject Sets from workflows") if not self._auto_disable_subject_sets: self.log.info( f"Configuration prevents from disabling Subject Sets automatically" ) return panoptes_client, project = self.get_conn() for wsum in workflows_summary: workflow = Workflow.find(wsum['id']) subject_sets = list() for ss in wsum['subject_sets']: if (ss['subjects_count'] != 0) and (ss['retired_count'] == ss['subjects_count']): subject_set = SubjectSet.find(ss['id']) subject_sets.append(subject_set) workflow.remove_subject_sets(subject_sets) self.log.info( f"Disabled: {len(subject_sets)} Subject Sets from '{workflow.display_name}'" )
def make_tutorial_images(imagePaths, ellipseData, projectData): # Connect to Panoptes Panoptes.connect( username=projectData["user_name"], password=projectData["password"] ) newSubjects = [] for imageId, imagePath in enumerate(imagePaths): print(f"Adding {imagePath}...") try: subjectSet = SubjectSet.find(projectData["subject_set"]) except PanoptesAPIException as e: print(e) return newSubject = Subject() newSubject.add_location(imagePath) newSubject.links.project = subjectSet.links.project newSubject.metadata.update( make_metadata( ellipseData.get_group(imageId).reset_index(drop=True), imagePath ) ) newSubject.save() newSubjects.append(newSubject) subjectSet.add(newSubjects)
def download_classifications(subject_set_id, output_file, generate, generate_timeout): """ Downloads a subject-set specific classifications export for the given subject set. OUTPUT_FILE will be overwritten if it already exists. Set OUTPUT_FILE to - to output to stdout. """ subject_set = SubjectSet.find(subject_set_id) if generate: click.echo("Generating new export...", err=True) export = subject_set.get_export('classifications', generate=generate, wait_timeout=generate_timeout) with click.progressbar( export.iter_content(chunk_size=1024), label='Downloading', length=(int(export.headers.get('content-length')) / 1024 + 1), file=click.get_text_stream('stderr'), ) as chunks: for chunk in chunks: output_file.write(chunk)
def push_new_row_subjects(self, source_subject, target_subject_set_id, row_paths_by_column): """ Given image paths for the new column-indexed rows (row_paths_by_column), push new unclassified row subjects to the appropriate subject set, with metadata references to the source subject and column. """ project = Project.find(settings.PROJECT_ID) subject_set_unclassified_rows = SubjectSet.find(target_subject_set_id) new_row_subjects = [] for column_index, row_paths in row_paths_by_column.items(): self._logger.info('Creating %d new row subjects for column index %d for subject %s', len(row_paths), column_index, source_subject.id) for row_path in row_paths: new_subject = Subject() new_subject.links.project = project copy_source_metadata_fields = ['book', 'page'] for copy_field in copy_source_metadata_fields: new_subject.metadata[copy_field] = source_subject.metadata[copy_field] new_subject.metadata['source_document_subject_id'] = source_subject.id new_subject.metadata['source_document_column_index'] = column_index new_subject.add_location(row_path) new_subject.save() new_row_subjects.append(new_subject) subject_set_unclassified_rows.add(new_row_subjects)
def ls(subject_set_id, project_id, workflow_id, quiet): """Lists subject set IDs and names""" if subject_set_id and not project_id and not workflow_id: subject_set = SubjectSet.find(subject_set_id) if quiet: click.echo(subject_set.id) else: echo_subject_set(subject_set) return args = {} if project_id: args['project_id'] = project_id if workflow_id: args['workflow_id'] = workflow_id if subject_set_id: args['subject_set_id'] = subject_set_id subject_sets = SubjectSet.where(**args) if quiet: click.echo(" ".join([s.id for s in subject_sets])) else: for subject_set in subject_sets: echo_subject_set(subject_set)
def ml_subject_assistant_export_to_microsoft_pt1_get_subjects_data( export_id, access_token, ): print('[Subject Assistant] Exporting to Microsoft 1/4: get Subjects') export = MLSubjectAssistantExport.objects.get(pk=export_id) data = [] # Keeps track of all data items that needs to written into a Microsoft-friendly JSON format. # Retrieve all Subjects from a Subject Set with SocialPanoptes(bearer_token=access_token) as p: subject_set = SubjectSet.find(export.subject_set_id) # Process each Subject for subject in subject_set.subjects: # Create a data item for each image URL in the Subject for frame_id, location in enumerate(subject.locations): image_url = list(location.values())[0] subject_information = { 'project_id': str(subject_set.links.project.id), 'subject_set_id': str(export.subject_set_id), 'subject_id': str(subject.id), 'frame_id': str(frame_id) } item = [] item.append(image_url) item.append(json.dumps(subject_information)) # The subject's JSON information is stored as a string. Yes, really. data.append(item) return data
def modify(subject_set_id, project_id, display_name): subject_set = SubjectSet.find(subject_set_id) if project_id: subject_set.links.project = project_id if display_name: subject_set.display_name = display_name subject_set.save() echo_subject_set(subject_set)
def add_to_subject_set(subject_set_id, subject_set_file): """Import a 1 column file of subject_ids to a subject_set.""" lines = [] subject_set = SubjectSet.find(subject_set_id) with open(subject_set_file) as subject_ids: lines.append(subject_ids.read().splitlines()) return subject_set.add(np.unique(lines))
def _get_subject_set(self, scope, project_id, set_name): subject_set = None if not scope.subject_set_id: subject_set = self._create_subject_set(project_id, set_name) scope.subject_set_id = subject_set.id scope.save() else: subject_set = SubjectSet.find(scope.subject_set_id) return subject_set
def remove_subject(self, subject_set_id, subject_list): """ :param subject_list: :return: """ subject_set = SubjectSet.find(subject_set_id) x = subject_set.remove(subject_list) y = subject_set.save() return x, y
def modify(subject_set_id, display_name): """ Changes the attributes of an existing subject set. Any attributes which are not specified are left unchanged. """ subject_set = SubjectSet.find(subject_set_id) if display_name: subject_set.display_name = display_name subject_set.save() echo_subject_set(subject_set)
def delete(force, subject_set_ids): for subject_set_id in subject_set_ids: subject_set = SubjectSet.find(subject_set_id) if not force: click.confirm( 'Delete subject set {} ({})?'.format( subject_set_id, subject_set.display_name, ), abort=True, ) subject_set.delete()
def remove_subjects(subject_set_id, subject_ids, id_file): """ Unlinks subjects from this subject set. The subjects themselves are not deleted or modified in any way and will still be present in any other sets they're linked to. """ s = SubjectSet.find(subject_set_id) if id_file: s.remove([l.strip() for l in id_file.readlines()]) if subject_ids: s.remove(subject_ids)
def add_subjects(subject_set_id, subject_ids, id_file): """ Links existing subjects to this subject set. This command is useful mainly for adding previously uploaded subjects to additional subject sets. See the upload-subjects command to create new subjects in a subject set. """ s = SubjectSet.find(subject_set_id) if id_file: s.add([l.strip() for l in id_file.readlines()]) if subject_ids: s.add(subject_ids)
def get_level_structure(self, workflow=None, IDfilter=''): """Parameters ---------- workflow : `int`, optional, default None IDfilter : `str`, optional, default '' Returns ------- A dict with keys of workflow IDs and values list of golden sets associated with that workflow """ if hasattr(self, 'level_structure'): return self.level_structure level_structure = {} workflowDictSubjectSets = \ self.get_subject_sets_per_workflow(workflow=workflow) for iworkflow in workflowDictSubjectSets.keys(): # If it is final workflow level 4 subject sets are also linked # so need to filter for level 5 subject sets if int(iworkflow) == 7767: subjectset_id = [iid for iid in \ workflowDictSubjectSets[iworkflow] \ if iid not in workflowDictSubjectSets['7766']] else: subjectset_id = workflowDictSubjectSets[iworkflow] # Determine Display names of subject set subjectset_displayname_id = {} for iSubjectSet in subjectset_id: tmp2 = SubjectSet.find(iSubjectSet) displayname = str(tmp2.raw['display_name']) if IDfilter in displayname: str_tmp = displayname.split(" '")[0].replace(' ', '_') subjectset_displayname_id[str_tmp] = \ (iworkflow, iSubjectSet, [float(iThres) for iThres in re.findall("\d+\.\d+", displayname) ] ) level_structure[iworkflow] = subjectset_displayname_id self.level_structure = level_structure return level_structure
def upload_subjects(subject_set_id, manifest_file): subject_set = SubjectSet.find(subject_set_id) subject_rows = [] with open(manifest_file) as manifest_f: file_root = os.path.dirname(manifest_file) r = csv.reader(manifest_f) headers = r.next() for row in r: metadata = dict(zip(headers, row)) files = [] for col in row: file_match = re.match(IMAGE_REGEX, col) file_path = os.path.join(file_root, col) if file_match and os.path.exists(file_path): files.append(file_path) if len(files) == 0: click.echo('Could not find any files in row:', err=True) click.echo(','.join(row), err=True) return -1 subject_rows.append((files, metadata)) created_subjects = [] with click.progressbar( enumerate(subject_rows), length=len(subject_rows), label='Uploading subjects', ) as _subject_rows: for count, (files, metadata) in _subject_rows: subject = Subject() subject.links.project = subject_set.links.project map(subject.add_location, files) subject.metadata.update(metadata) subject.save() created_subjects.append(subject) if (count + 1) % LINK_BATCH_SIZE == 0: subject_set.add(created_subjects) created_subjects = [] if len(created_subjects) > 0: subject_set.add(created_subjects)
def get_golden_images(self, workflow=None): """Parameters ---------- Returns ------- A dict with keys of workflow IDs and values list of golden sets associated with that workflow """ workflowGoldenSetImagesDict = {} workflowGoldenSetDict = self.get_golden_subject_sets() if workflow: workflows = workflow else: workflows = workflowGoldenSetDict.keys() for iWorkflow in workflows: goldenImages = {} for iGoldenSubjectSet in workflowGoldenSetDict[iWorkflow]: tmp = SubjectSet.find(iGoldenSubjectSet) tmpSubjects = tmp.subjects while True: try: nextSubject = tmpSubjects.next() goldenImages[str(nextSubject.id)] = \ [str(nextSubject.raw['metadata']['subject_id']),\ str(nextSubject.raw['metadata']['#Label']), int(nextSubject.id)] except: break workflowGoldenSetImagesDict[iWorkflow] = goldenImages self.workflowGoldenSetImagesDict = workflowGoldenSetImagesDict return workflowGoldenSetImagesDict
imageio.imwrite('crop.jpeg', image[:,160:1765,:]) # bit rate #(img) C:/Users/Rdebbout/Downloads/vids_DUL>ffmpeg -i test_out4.mp4 -b 1397520 bit_down2.mp4 # resolution #(img) C:/Users/Rdebbout/Downloads/vids_DUL>ffmpeg -i test_out4.mp4 -vf scale=960:540 bit_down_scale.mp4 # ffprobe -v quiet -print_format json -show_format -show_streams test_out4.mp4 > op.json ################################################################################ from panoptes_client import SubjectSet, Subject, Project, Panoptes Panoptes.connect(username='******', password='******') project = Project.find(id = 5483) subject_set = SubjectSet.find(17639) subject = Subject() subject.links.project = project subject.add_location({'video/mp4': ('C:/Users/Rdebbout/Downloads/vids_DUL/' 'test_frame_rate/duo_DVR150925_1432_001clip.mp4')}) subject.metadata['site_id'] = 'NCCAGL10-1047' subject.save() subject_set.add(subject) ################################################################################ here = 'C:/Users/Rdebbout/Downloads/vids_DUL/test_frame_rate/prepare_ye' tbl_list = pd.read_csv('CitSci_VideoList_beta.csv') for f in os.listdir(here): print f
def make_subject_fits( full_subject_set_id, center_workflow_id, spiral_workflow_id, bar_workflow_id, dimensions=[525, 525], image_location='/Volumes/SD_Extra/manga_images_production/MPL5', output='MPL5_fits'): blank_mask = np.zeros(dimensions) coords = [[x, y] for y in xrange(dimensions[1]) for x in xrange(dimensions[0])] fullsample = SubjectSet.find(full_subject_set_id) subjects = fullsample.subjects() pbar = pb.ProgressBar(widgets=widgets, maxval=subjects.meta['count']) pbar.start() idx = 0 for subject in subjects: subject_metadata = Table(dtype=metadata_dtype) subject_metadata.add_row( tuple(subject.raw['metadata'][key] for key in subject_metadata.dtype.names)) subject_metadata.rename_column('#MANGA_TILEID', 'MANGA_TILEID') subject_metadata_hdu = fits.table_to_hdu(subject_metadata) loc = '{0}/{1}_{2}.jpg'.format( image_location, subject_metadata['MANGAID'][0], int(subject_metadata['IFUDESIGNSIZE'][0])) output_name = '{0}/{1}_{2}_{3}.fits'.format( output, subject_metadata['MANGAID'][0], int(subject_metadata['IFUDESIGNSIZE'][0]), subject.id) if os.path.isfile('{0}.gz'.format(output_name)): # don't process the file if it already exists idx += 1 pbar.update(idx) continue image = plt.imread(loc, format='jpeg') wcs = define_wcs(subject_metadata['ra'][0], subject_metadata['dec'][0]) wcs_header = wcs.to_header() orig_image_hdu = fits.PrimaryHDU(data=image, header=wcs_header) # process data from center(s) and star(s) points center_classifications = make_classification_table( 'center_points', 'star_points') all_center = [] all_star = [] for c in Classification.where(scope='project', workflow_id=center_workflow_id, subject_id=subject.id): record_base_classification(c, center_classifications) center_points = [] star_points = [] points = c.raw['annotations'][0]['value'] for p in points: if ('x' in p) and ('y' in p): if p['tool'] == 0: # somehow the workflow_id got messed up for some classifications # so a try statement is needed loc = [p['x'], p['y']] center_points.append(loc) all_center.append(loc) elif p['tool'] == 1: loc = [p['x'], p['y']] star_points.append(loc) all_star.append(loc) center_classifications['center_points'].append( json.dumps(center_points)) center_classifications['star_points'].append( json.dumps(star_points)) center_star_table_hdu = fits.table_to_hdu( Table(center_classifications)) # cluster points and make image masks if len(all_center): center_mask, center_table_hdu = cluster(np.array(all_center), dimensions, coords, wcs) else: center_mask = blank_mask center_table_hdu = fits.table_to_hdu(Table(make_cluster_table())) center_hdu = fits.ImageHDU(data=center_mask, header=wcs_header) if len(all_star): star_mask, star_table_hdu = cluster(np.array(all_star), dimensions, coords, wcs) else: star_mask = blank_mask star_table_hdu = fits.table_to_hdu(Table(make_cluster_table())) star_hdu = fits.ImageHDU(data=star_mask, header=wcs_header) # spiral arms spiral_table_hdu, spiral_hdu = mask_process(spiral_workflow_id, subject.id, 'spiral_paths', wcs_header, dimensions, coords) # bars bar_table_hdu, bar_hdu = mask_process(bar_workflow_id, subject.id, 'bar_paths', wcs_header, dimensions, coords) # make fits file hdu_list = fits.HDUList([ orig_image_hdu, center_hdu, star_hdu, spiral_hdu, bar_hdu, subject_metadata_hdu, center_table_hdu, star_table_hdu, center_star_table_hdu, spiral_table_hdu, bar_table_hdu ]) hdu_list.writeto(output_name) # compress the fits file call(['gzip', output_name]) # update progressbar idx += 1 pbar.update(idx) pbar.finish()
def ls(subject_set_id): echo_subject_set(SubjectSet.find(subject_set_id))
def upload_subjects( subject_set_id, manifest_files, allow_missing, remote_location, mime_type, file_column, ): """ Uploads subjects from each of the given MANIFEST_FILES. Example with only local files: $ panoptes subject-set upload-subjects 4667 manifest.csv Local filenames will be automatically detected in the manifest and uploaded, or filename columns can be specified with --file-column. If you are hosting your media yourself, you can put the URLs in the manifest and specify the column number(s): $ panoptes subject-set upload-subjects -r 1 4667 manifest.csv $ panoptes subject-set upload-subjects -r 1 -r 2 4667 manifest.csv Any local files will still be detected and uploaded. """ if ( len(manifest_files) > 1 and any(map(lambda m: m.endswith('.yaml'), manifest_files)) ): click.echo( 'Error: YAML manifests must be processed one at a time.', err=True, ) return -1 elif manifest_files[0].endswith('.yaml'): with open(manifest_files[0], 'r') as yaml_manifest: upload_state = yaml.load(yaml_manifest, Loader=yaml.FullLoader) if upload_state['state_version'] > CURRENT_STATE_VERSION: click.echo( 'Error: {} was generated by a newer version of the Panoptes ' 'CLI and is not compatible with this version.'.format( manifest_files[0], ), err=True, ) return -1 if upload_state['subject_set_id'] != subject_set_id: click.echo( 'Warning: You specified subject set {} but this YAML ' 'manifest is for subject set {}.'.format( subject_set_id, upload_state['subject_set_id'], ), err=True, ) click.confirm( 'Upload {} to subject set {} ({})?'.format( manifest_files[0], subject_set_id, SubjectSet.find(subject_set_id).display_name, ), abort=True ) upload_state['subject_set_id'] = subject_set_id resumed_upload = True else: upload_state = { 'state_version': CURRENT_STATE_VERSION, 'subject_set_id': subject_set_id, 'manifest_files': manifest_files, 'allow_missing': allow_missing, 'remote_location': remote_location, 'mime_type': mime_type, 'file_column': file_column, 'waiting_to_upload': [], 'waiting_to_link': {}, } resumed_upload = False remote_location_count = len(upload_state['remote_location']) mime_type_count = len(upload_state['mime_type']) if remote_location_count > 1 and mime_type_count == 1: upload_state['mime_type'] = ( upload_state['mime_type'] * remote_location_count ) elif remote_location_count > 0 and mime_type_count != remote_location_count: click.echo( 'Error: The number of MIME types given must be either 1 or equal ' 'to the number of remote locations.', err=True, ) return -1 def validate_file(file_path): if not os.path.isfile(file_path): click.echo( 'Error: File "{}" could not be found.'.format( file_path, ), err=True, ) return False file_size = os.path.getsize(file_path) if file_size == 0: click.echo( 'Error: File "{}" is empty.'.format( file_path, ), err=True, ) return False elif file_size > MAX_UPLOAD_FILE_SIZE: click.echo( 'Error: File "{}" is {}, larger than the maximum {}.'.format( file_path, humanize.naturalsize(file_size), humanize.naturalsize(MAX_UPLOAD_FILE_SIZE), ), err=True, ) return False return True subject_set = SubjectSet.find(upload_state['subject_set_id']) if not resumed_upload: subject_rows = [] for manifest_file in upload_state['manifest_files']: with open(manifest_file, 'U') as manifest_f: file_root = os.path.dirname(manifest_file) r = csv.reader(manifest_f, skipinitialspace=True) headers = next(r) for row in r: metadata = dict(zip(headers, row)) files = [] if not upload_state['file_column']: upload_state['file_column'] = [] for field_number, col in enumerate(row, start=1): file_path = os.path.join(file_root, col) if os.path.exists(file_path): upload_state['file_column'].append( field_number, ) if not validate_file(file_path): return -1 files.append(file_path) else: for field_number in upload_state['file_column']: file_path = os.path.join( file_root, row[field_number - 1] ) if not validate_file(file_path): return -1 files.append(file_path) for field_number, _mime_type in zip( upload_state['remote_location'], upload_state['mime_type'], ): files.append({_mime_type: row[field_number - 1]}) if len(files) == 0: click.echo( 'Could not find any files in row:', err=True, ) click.echo(','.join(row), err=True) if not upload_state['allow_missing']: return -1 else: continue subject_rows.append((files, metadata)) if not subject_rows: click.echo( 'File {} did not contain any rows.'.format( manifest_file, ), err=True, ) return -1 subject_rows = list(enumerate(subject_rows)) upload_state['waiting_to_upload'] = copy.deepcopy(subject_rows) else: for subject_id, subject_row in upload_state['waiting_to_link'].items(): try: subject = Subject.find(subject_id) except PanoptesAPIException: upload_state['waiting_to_upload'].append(subject_row) del upload_state['waiting_to_link'][subject_id] subject_rows = copy.deepcopy(upload_state['waiting_to_upload']) pending_subjects = [] def move_created(limit): while len(pending_subjects) > limit: for subject, subject_row in pending_subjects: if subject.async_save_result: pending_subjects.remove((subject, subject_row)) upload_state['waiting_to_upload'].remove(subject_row) upload_state['waiting_to_link'][subject.id] = subject_row time.sleep(0.5) def link_subjects(limit): if len(upload_state['waiting_to_link']) > limit: subject_set.add(list(upload_state['waiting_to_link'].keys())) upload_state['waiting_to_link'].clear() with click.progressbar( subject_rows, length=len(subject_rows), label='Uploading subjects', ) as _subject_rows: try: with Subject.async_saves(): for subject_row in _subject_rows: count, (files, metadata) = subject_row subject = Subject() subject.links.project = subject_set.links.project for media_file in files: subject.add_location(media_file) subject.metadata.update(metadata) subject.save() pending_subjects.append((subject, subject_row)) move_created(MAX_PENDING_SUBJECTS) link_subjects(LINK_BATCH_SIZE) move_created(0) link_subjects(0) finally: if ( len(pending_subjects) > 0 or len(upload_state['waiting_to_link']) > 0 ): click.echo('Error: Upload failed.', err=True) if click.confirm( 'Would you like to save the upload state to resume the ' 'upload later?', default=True, ): while True: state_file_name = 'panoptes-upload-{}.yaml'.format( subject_set_id, ) state_file_name = click.prompt( 'Enter filename to save to', default=state_file_name, ) if not state_file_name.endswith('.yaml'): click.echo( 'Error: File name must end in ".yaml".', err=True, ) if click.confirm( 'Save to {}.yaml?'.format(state_file_name), default=True, ): state_file_name += '.yaml' else: continue if not is_valid_filename(state_file_name): click.echo( 'Error: {} is not a valid file name'.format( state_file_name, ), err=True, ) sanitized_filename = sanitize_filename( state_file_name, ) if click.confirm( 'Save to {}?'.format( sanitized_filename, ), default=True, ): state_file_name = sanitized_filename else: continue if os.path.exists(state_file_name): if not click.confirm( 'File {} already exists. Overwrite?'.format( state_file_name, ), default=False, ): continue break with open(state_file_name, 'w') as state_file: yaml.dump(upload_state, state_file)
subject.links.project = project subject.add_location(file) # You can set whatever metadata you want, or none at all subject.metadata['filename'] = os.path.basename(file) #TODO subject.metadata['file_start'] = #TODO subject.metadata['sample_rate'] = 5512 subject.metadata['fft'] = fft subject.metadata['overlap'] = overlap subject.metadata['color_min'] = color_min subject.metadata['color_max'] = color_max #TODO subject.metadata['width'] = #TODO subject.metadata['height'] = subject.save() subjects.append(subject) os.rename(file,dest+os.path.basename(file)) #move file to uploaded directory #Create a new subject set or append the subjects to an existing one for subject_set in project.links.subject_sets: if str(subject_set.display_name) == subject_set_display_name: subject_set_id = subject_set.id subject_set = SubjectSet.find(subject_set_id) break else: #subject_set = SubjectSet() #subject_set.links.project = project #subject_set.display_name = subject_set_display_name #subject_set.save() raise Exception('Subject set does not exist') subject_set.add(subjects) # SubjectSet.add_subjects() can take a list of Subjects, or just one. print("--- %s seconds ---" % (time.time() - start_time))
args = parser.parse_args() username = input(f"Enter Zooiverse username:"******"Enter Zooniverse password for {username}: ") if args.password is None else args.password ) projectData = { "subject_set": args.subjectSetId, "user_name": username, "password": password, } imagePaths = pd.read_csv(args.imagePaths, header=None)[0] ellipseData = pd.read_csv(args.ellipseData).groupby(by="image_id") try: print( "Adding subjects to {} {}".format( SubjectSet.find(projectData["subject_set"]).display_name, projectData["subject_set"], ) ) except PanoptesAPIException as e: print(e) exit(1) make_tutorial_images(imagePaths, ellipseData, projectData)
and add the datetime to the subject metadata. It requires the project owner credentials to be set up as OS environmental variables, and an appropriate project slug modified on line 11. depending on the camera used to take the original subject image the exif code may be different than that in the code and may need to be modified""" import os from PIL import Image, ExifTags import panoptes_client from panoptes_client import SubjectSet, Project, Panoptes import requests Panoptes.connect(username=os.environ['User_name'], password=os.environ['Password']) project = Project.find(slug='pmason\fossiltrainer') while True: set_id = input('Entry subject set id to update:' + '\n') try: subject_set = SubjectSet.find(set_id) count_subjects = 0 subject_list = [] for subject in subject_set.subjects: count_subjects += 1 if subject.metadata['DateTime'] == '': try: img = Image.open(requests.get(subject.locations[0]['image/jpeg'], stream=True).raw) exif_dict = img._getexif() date_time = exif_dict[306] except (IOError, KeyError): print('Acquiring exif data for ', subject.id, ' failed') continue subject.metadata['DateTime'] = date_time print(subject.id, subject.metadata['DateTime'], ' fixed') subject.save()
# 1) Edit input parameters below. # 2) Run from command line: `python edit_metadata.py` from panoptes_client import Panoptes, SubjectSet ################################### # Input Parameters puser = '******' ppswd = 'PASSWORD' subject_set_ids = [77172] new_metadata = { "Attribution": "Photos are open source under a CC BY 3.0 license and were contributed to the project via CitSci.org." } ################################### Panoptes.connect(username=puser, password=ppswd) for ssid in subject_set_ids: subject_set = SubjectSet.find(ssid) print('Editing metadata for Subject Set #{0}: {1}'.format( subject_set.raw['id'], subject_set.raw['display_name'])) for subject in subject_set.subjects: for key, value in new_metadata.items(): subject.metadata[key] = value subject.save()
def info(subject_set_id): subject_set = SubjectSet.find(subject_set_id) click.echo(yaml.dump(subject_set.raw))
def _get_subject_set(self, subject_set_id): if subject_set_id not in self._subject_sets: self._subject_sets[subject_set_id] = SubjectSet.find( subject_set_id) return self._subject_sets[subject_set_id]
4) Remove the second set of RA and DEC columns, and name the first ones 'RA' and 'DEC' 5) Save this file as a csv 6) This only works in python 2 because it uses panoptes_client ''' import numpy as np from panoptes_client import SubjectSet, Subject, Project, Panoptes import os import progressbar as pb myusername = os.environ['PANOPTES_USERNAME'] mypassword = os.environ['PANOPTES_PASSWORD'] Panoptes.connect(username= myusername, password=mypassword) project = Project.find(id='73') fullsample = SubjectSet.find(5326) spirals = SubjectSet.find(5324) bars = SubjectSet.find(5325) progress = pb.ProgressBar(widgets=[pb.Bar(), pb.ETA()]) data = np.genfromtxt('../GZ3D/MatchedData.csv', delimiter = ',', names=True, dtype=[('DEC', float), ('IAUNAME', '|S30'),('IFUTARGETSIZE',int), ('MANGAID', '|S10'),('MANGA_TILEID',int),('NSAID', int), ('PETROTH50',float),('RA',float),('SERSIC_TH50',float), ('Z',float),('specobjid', int),('dr8objid', int), ('dr7objid', int),('t01_smooth_or_features_a02_features_or_disk_weighted_fraction', float), ('t02_edgeon_a05_no_weighted_fraction', float), ('t03_bar_a06_bar_weighted_fraction', float), ('t04_spiral_a08_spiral_weighted_fraction', float)]) counter = 0 nancounter = 0 spiralcount = 0
#!/usr/bin/env python3 import sys sys.path.insert(0, "..") import pdb from panoptes_client import Panoptes, SubjectSet from lib import settings Panoptes.connect(username=settings.PANOPTES_USERNAME, password=settings.PANOPTES_PASSWORD) subject_set = SubjectSet.find(14804) print(' '.join([s.id for s in subject_set.subjects]))