def main(): parser = argparse.ArgumentParser( description='Dump languages distribution by article') parser.add_argument('issns', nargs='*', help='ISSN\'s separated by spaces') parser.add_argument('--collection', '-c', help='Collection Acronym') parser.add_argument('--output_file', '-r', help='File to receive the dumped data') parser.add_argument('--logging_file', '-o', help='Full path to the log file') parser.add_argument( '--logging_level', '-l', default='DEBUG', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='Logggin level') args = parser.parse_args() _config_logging(args.logging_level, args.logging_file) logger.info('Dumping data for: %s' % args.collection) issns = None if len(args.issns) > 0: issns = utils.ckeck_given_issns(args.issns) dumper = Dumper(args.collection, issns, args.output_file) dumper.run()
def main(): parser = argparse.ArgumentParser( description='Dump languages distribution by article' ) parser.add_argument( 'issns', nargs='*', help='ISSN\'s separated by spaces' ) parser.add_argument( '--collection', '-c', help='Collection Acronym' ) parser.add_argument( '--not_normalized', '-n', action='store_true', help='Dump only not normalized affiliations' ) parser.add_argument( '--output_file', '-r', help='File to receive the dumped data' ) parser.add_argument( '--logging_file', '-o', help='Full path to the log file' ) parser.add_argument( '--logging_level', '-l', default='DEBUG', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='Logggin level' ) args = parser.parse_args() _config_logging(args.logging_level, args.logging_file) logger.info('Dumping data for: %s' % args.collection) issns = None if len(args.issns) > 0: issns = utils.ckeck_given_issns(args.issns) dumper = Dumper(args.collection, issns, args.output_file, args.not_normalized) dumper.run()
def main(): parser = argparse.ArgumentParser( description='Update the citing numbers of each documents in search.scielo.org' ) parser.add_argument( 'issns', nargs='*', help='ISSN\'s separated by spaces' ) parser.add_argument( '--collection', '-c', help='Collection Acronym' ) parser.add_argument( '--citations_mode', '-m', default='pid', choices=['meta', 'pid'], help='Mode to retrieve received citations.' ) parser.add_argument( '--logging_file', '-o', help='Full path to the log file' ) parser.add_argument( '--logging_level', '-l', default='DEBUG', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='Logggin level' ) args = parser.parse_args() _config_logging(args.logging_level, args.logging_file) logger.info('Dumping data for: %s' % args.collection) issns = None if len(args.issns) > 0: issns = utils.ckeck_given_issns(args.issns) dumper = Dumper(args.collection, issns, args.citations_mode) dumper.run()
def main(): parser = argparse.ArgumentParser( description='Importa metadados de catalogos SciELO do modelo legado para o SciELO Manager' ) parser.add_argument( 'issns', nargs='*', help='ISSN\'s separated by spaces' ) parser.add_argument( '--collection_from', '-c', help='Collection Acronym' ) parser.add_argument( '--collection_to', '-t', help='Collection Acronym' ) parser.add_argument( '--logging_file', '-o', help='Full path to the log file' ) parser.add_argument( '--logging_level', '-l', default='DEBUG', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='Logggin level' ) args = parser.parse_args() _config_logging(args.logging_level, args.logging_file) logger.info(u'Importing data from collection: %s' % args.collection_from) issns = None if len(args.issns) > 0: issns = utils.ckeck_given_issns(args.issns) try: importer = Importer(args.collection_from, args.collection_to, issns=issns) except ValueError: logger.error(u'Destiny collection do not exists (%s)' % args.collection_to) else: importer.run()
def main(): parser = argparse.ArgumentParser( description='Export journals list in Kabart format' ) parser.add_argument( 'issns', nargs='*', help='ISSN\'s separated by spaces' ) parser.add_argument( '--collection', '-c', help='Collection Acronym' ) parser.add_argument( '--output_file', '-r', help='File to receive the dumped data' ) parser.add_argument( '--logging_file', '-o', help='Full path to the log file' ) parser.add_argument( '--logging_level', '-l', default='DEBUG', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='Logggin level' ) args = parser.parse_args() _config_logging(args.logging_level, args.logging_file) logger.info('Dumping data for: %s' % args.collection) issns = None if len(args.issns) > 0: issns = utils.ckeck_given_issns(args.issns) dumper = Dumper(args.collection, issns, args.output_file) dumper.run()
def main(): parser = argparse.ArgumentParser( description="Dump SciELO Network metadata") parser.add_argument('issns', nargs='*', help='ISSN\'s separated by spaces') parser.add_argument('--collection', '-c', help='Collection Acronym') parser.add_argument( '--zip_file', '-f', default='dumpdata.zip', help='Full path to the zip file that will receive the documents') parser.add_argument('--xml_format', '-x', default='xmlwos', choices=['xmlwos', 'xmlrsps'], help='XML output format') parser.add_argument('--logging_file', '-o', help='Full path to the log file') parser.add_argument( '--logging_level', '-l', default='DEBUG', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='Logggin level') args = parser.parse_args() _config_logging(args.logging_level, args.logging_file) logger.info('Dumping data for: %s' % args.collection) issns = None if len(args.issns) > 0: issns = utils.ckeck_given_issns(args.issns) dumper = Dumper(args.collection, issns, args.xml_format, args.zip_file) dumper.run()
def main(): parser = argparse.ArgumentParser(description='Dump SciELO tabs') parser.add_argument('issns', nargs='*', help='ISSN\'s separated by spaces') parser.add_argument('--collection', '-c', help='Collection Acronym') parser.add_argument( '--home_nationality', '-n', help= 'ISO 3166 two letters country code which will be considered as the home nationality.' ) parser.add_argument('--logging_file', '-o', help='Full path to the log file') parser.add_argument( '--logging_level', '-l', default='DEBUG', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='Logggin level') args = parser.parse_args() _config_logging(args.logging_level, args.logging_file) logger.info('Dumping data for: %s' % args.collection) issns = None if len(args.issns) > 0: issns = utils.ckeck_given_issns(args.issns) dumper = Dumper(args.collection, home_nationality=args.home_nationality, issns=issns) dumper.run()
def main(): parser = argparse.ArgumentParser( description='Dump accesses' ) parser.add_argument( 'issns', nargs='*', help='ISSN\'s separated by spaces' ) parser.add_argument( '--collection', '-c', help='Collection Acronym' ) parser.add_argument( '--dayly_granularity', '-d', action='store_true', help='Accesses granularity default will be monthly if not specified' ) parser.add_argument( '--from_date', '-b', default=FROM, help='Delimite the accesses start period' ) parser.add_argument( '--until_date', '-u', default=UNTIL, help='Delimite the accesses end period' ) parser.add_argument( '--output_format', '-f', choices=['json', 'csv'], default=OUTPUT_FORMAT, help='Output format' ) parser.add_argument( '--output_file', '-r', help='File to receive the dumped data' ) parser.add_argument( '--logging_file', '-o', help='Full path to the log file' ) parser.add_argument( '--logging_level', '-l', default='DEBUG', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='Logggin level' ) args = parser.parse_args() _config_logging(args.logging_level, args.logging_file) logger.info('Dumping data for: %s' % args.collection) issns = None if len(args.issns) > 0: issns = utils.ckeck_given_issns(args.issns) if not utils.is_valid_date(args.from_date): logger.error('Invalid from date: %s' % args.from_date) exit() if not utils.is_valid_date(args.until_date): logger.error('Invalid until date: %s' % args.until_date) exit() dumper = Dumper(args.collection, issns, args.from_date, args.until_date, args.dayly_granularity, args.output_format, args.output_file) dumper.run()
def main(): parser = argparse.ArgumentParser( description='Load documents into DOAJ' ) parser.add_argument( 'issns', nargs='*', help='ISSN\'s separated by spaces' ) parser.add_argument( '--issns_file', '-i', default=None, help='Full path to a txt file within a list of ISSNs to be exported' ) parser.add_argument( '--user', '-u', required=True, help='DOAJ Publisher account name' ) parser.add_argument( '--password', '-p', required=True, help='DOAJ Publisher account password' ) parser.add_argument( '--collection', '-c', help='Collection Acronym' ) parser.add_argument( '--from_date', '-f', default=FROM, help='ISO date like %s' % FROM ) parser.add_argument( '--logging_file', '-o', help='Full path to the log file' ) parser.add_argument( '--logging_level', '-l', default='DEBUG', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='Logggin level' ) args = parser.parse_args() _config_logging(args.logging_level, args.logging_file) logger.info('Dumping data for: %s' % args.collection) issns = None if len(args.issns) > 0: issns = utils.ckeck_given_issns(args.issns) issns_from_file = None if args.issns_file: with open(args.issns_file, 'r') as f: issns_from_file = utils.ckeck_given_issns([i.strip() for i in f]) if issns: issns += issns_from_file if issns_from_file else [] else: issns = issns_from_file if issns_from_file else [] dumper = Dumper( args.collection, issns, from_date=args.from_date, user=args.user, password=args.password) dumper.run()
def test_ckeck_given_issns(self): result = utils.ckeck_given_issns(['1234-4321', '1244-333x']) self.assertEqual(sorted(result), sorted(['1234-4321', '1244-333x']))
def test_ckeck_given_issns_empty(self): result = utils.ckeck_given_issns([]) self.assertEqual(result, [])
def test_ckeck_given_issns_with_invalid_issn(self): result = utils.ckeck_given_issns(['1234-432a', '1244-333x']) self.assertEqual(result, ['1244-333x'])
def main(): parser = argparse.ArgumentParser( description='Dump languages distribution by article' ) parser.add_argument( 'issns', nargs='*', help='ISSN\'s separated by spaces' ) parser.add_argument( '--issns_file', '-i', default=None, help='Full path to a txt file within a list of ISSNs to be exported' ) parser.add_argument( '--collection', '-c', help='Collection Acronym' ) parser.add_argument( '--processes', '-p', type=int, default=1, help='Number of processes per CPU' ) parser.add_argument( '--logging_file', '-o', help='Full path to the log file' ) parser.add_argument( '--logging_level', '-l', default='DEBUG', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='Logggin level' ) args = parser.parse_args() _config_logging(args.logging_level, args.logging_file) logger.info('Dumping data for: %s' % args.collection) issns = None if len(args.issns) > 0: issns = utils.ckeck_given_issns(args.issns) issns_from_file = None if args.issns_file: with open(args.issns_file, 'r') as f: issns_from_file = utils.ckeck_given_issns([i.strip() for i in f]) if issns: issns += issns_from_file if issns_from_file else [] else: issns = issns_from_file if issns_from_file else [] dumper = Dumper(args.collection, issns) dumper.run(args.processes)
def main(): parser = argparse.ArgumentParser(description='Load documents into DOAJ') parser.add_argument('issns', nargs='*', help='ISSN\'s separated by spaces') parser.add_argument( '--issns_file', '-i', default=None, help='Full path to a txt file within a list of ISSNs to be exported') parser.add_argument('--user', '-u', required=True, help='DOAJ Publisher account name') parser.add_argument('--password', '-p', required=True, help='DOAJ Publisher account password') parser.add_argument('--collection', '-c', help='Collection Acronym') parser.add_argument('--from_date', '-f', default=FROM, help='ISO date like %s' % FROM) parser.add_argument('--logging_file', '-o', help='Full path to the log file') parser.add_argument( '--logging_level', '-l', default='DEBUG', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='Logggin level') args = parser.parse_args() _config_logging(args.logging_level, args.logging_file) logger.info('Dumping data for: %s' % args.collection) issns = None if len(args.issns) > 0: issns = utils.ckeck_given_issns(args.issns) issns_from_file = None if args.issns_file: with open(args.issns_file, 'r') as f: issns_from_file = utils.ckeck_given_issns([i.strip() for i in f]) if issns: issns += issns_from_file if issns_from_file else [] else: issns = issns_from_file if issns_from_file else [] dumper = Dumper(args.collection, issns, from_date=args.from_date, user=args.user, password=args.password) dumper.run()