def _add(index, submission_paths, only_these=None): from aggregator.record_aggregator import RecordAggregator record_aggregator = RecordAggregator(index) submission_label = Label(min_length=10) pbar = AlwaysUpdatingProgressBar(maxval=len(submission_paths), widgets=[ Percentage(), submission_label, Bar(marker='#', left='[', right=']') ]).start() for i, submission_path in enumerate(submission_paths): shared_dcontext.dcontext.submission = os.path.basename(submission_path) submission_label.change_text(shared_dcontext.dcontext.submission) pbar.update(i) if only_these is not None: inspire_id = int( shared_dcontext.dcontext.submission.replace('ins', '')) if inspire_id not in only_these: continue try: record_aggregator.process_submission(submission_path) except TransportError as err: print(err) print(dir(err)) raise err pbar.finish() print('Done', file=sys.stderr) record_aggregator.report_statistics()
def add_demo_mini(): # Add a couple of fake publications, useful to test ElasticSearch queries from aggregator.record_aggregator import RecordAggregator record_aggregator = RecordAggregator('hepdata-mini-demo') record_aggregator.load_mini_demo() record_aggregator.report_statistics()