#!/usr/bin/env python3 from datamodel_parser.application import Argument from datamodel_parser.application import Store from sdssdb.sqlalchemy.archive.sas import * from json import dumps import time print('Populating filespec table.') arg = Argument('filespec_archive') options = arg.options if arg else None store = Store(options=options) if options else None store.set_database() logger = store.logger if store else None ready = options and store and logger ready = ready and store.ready and store.database and store.database.ready if not ready: print('Fail! ready: {}'.format(ready)) exit(1) else: store.set_tree_edition() store.set_filespec() store.set_filepaths() store.populate_filespec_table_archive() store.exit() ''' Examples: filespec_archive.py --level debug --verbose --test --limit 1000 --path MANGA_SPECTRO_ANALYSIS/DRPVER/DAPVER/dapall.html --location manga/spectro/analysis/v2_4_3/2.2.1
#!/usr/bin/env python3 from datamodel_parser.application import Argument from datamodel_parser.application import Store print('\nInverse Parsing HTML.') arg = Argument('inverse_parse_html') options = arg.options if arg else None store = Store(options=options) if options else None logger = store.logger if store else None ready = options and store and logger ready = ready and store.ready if not ready: print('Fail! ready: {}'.format(ready)) exit(1) else: print('path: {}'.format(options.path)) store.set_tree_edition() store.set_path(path=options.path) store.split_path() #store.get_db_column_tags() store.get_db_keyword_tags() store.exit() ''' inverse_parse_html -l error -v --path MANGA_SPECTRO_REDUX/DRPVER/PLATE4/stack/manga-CUBE.html --string VERSPLDS '''
#!/usr/bin/env python3 from os import walk from os.path import join, exists from datamodel_parser.application import Argument from datamodel_parser.application import Store from json import dumps print('Populating File Path Tables') arg = Argument('populate_path_tables') options = arg.options if arg else None store = Store(options=options) if options else None logger = store.logger if store else None ready = options and store and logger ready = ready and store.ready if not ready: print('Fail! ready: {}'.format(ready)) exit(1) else: store.set_datamodel_dir() store.set_tree_edition() store.set_database() store.set_filepaths() # print('store.filepaths: \n' + dumps(store.filepaths,indent=1)) # print('len(store.filepaths): \n' + dumps(len(store.filepaths),indent=1)) # input('pause') file_list = list() for path in store.filepaths: if store.ready: # if ('BOSS_SPECTRO_REDUX' in path or # 'MANGA_SPECTRO_REDUX' in path or
#!/usr/bin/env python3 from datamodel_parser.application import Argument from datamodel_parser.application import Store from json import dumps print('Populating filespec table.') arg = Argument('filespec_db') options = arg.options if arg else None store = Store(options=options) if options else None store.set_database() logger = store.logger if store else None ready = options and store and logger ready = ready and store.ready and store.database and store.database.ready if not ready: print('Fail! ready: {}'.format(ready)) exit(1) else: store.set_tree_edition() store.set_filepaths() store.util.set_yaml_attr(attr_obj=store, attr_name='filespec_dict_yaml', filename='filespec.yaml') store.populate_filespec_table_yaml() store.exit()
#!/usr/bin/env python3 from datamodel_parser.application import Argument from datamodel_parser.application import Store from json import dumps print('Creating yaml file with directory substitutions') arg = Argument('filespec_init') options = arg.options if arg else None store = Store(options=options) if options else None logger = store.logger if store else None ready = options and store and logger ready = ready and store.ready if not ready: print('Fail! ready: {}'.format(ready)) exit(1) else: store.init_filename_substitutions_yaml() store.exit()
#!/usr/bin/env python3 from os import walk from os.path import join, exists from datamodel_parser.application import Argument from datamodel_parser.application import Store from json import dumps print('Finding SVN Products') arg = Argument('find_svn_products') options = arg.options if arg else None store = Store(options=options) if options else None logger = store.logger if store else None ready = options and store and logger ready = ready and store.ready if not ready: print('Fail! ready: {}'.format(ready)) exit(1) else: store.svn_products = list() root_dir = 'https://trac.sdss.org/browser/repo/' store.set_svn_products(root_dir=root_dir) print('store.svn_products: \n' + dumps(store.svn_products, indent=1)) store.exit() ''' https://trac.sdss.org/browser/repo/apogee/apogeedb https://trac.sdss.org/browser/repo/apogee/apogeelinelist https://trac.sdss.org/browser/repo/apogee/apogeereduce https://trac.sdss.org/browser/repo/apogee/apogeesim https://trac.sdss.org/browser/repo/apogee/apogeetarget https://trac.sdss.org/browser/repo/apogee/aporbit