def run(config, filename, debug, limit, offset, resource): sys.stdout.write(filename + ' ... ') sys.stdout.flush() configparser = metl.configparser.ConfigParser( metl.config.Config(config), debug=debug, limit=limit, offset=offset, source_resource=resource) metl.manager.Manager(configparser.getTarget()).run() sys.stdout.write('OK\n') sys.stdout.flush()
def write(filepath, records): source = metl.source.staticsource.StaticSource( metl.fieldset.FieldSet(fields=[ metl.field.Field( 'key', metl.fieldtype.stringfieldtype.StringFieldType()) ], fieldmap=metl.fieldmap.FieldMap({'key': 0}))) source.setResource(sourceRecords=[[r] for r in records]) configparser = metl.configparser.ConfigParser( metl.config.Config(filepath), init_on_start=False) configparser.readers = [source] configparser.loadTarget() metl.manager.Manager(configparser.getTarget()).run()
def run( config, filename, debug, limit, offset, resource ): sys.stdout.write( filename + ' ... ' ) sys.stdout.flush() configparser = metl.configparser.ConfigParser( metl.config.Config( config ), debug = debug, limit = limit, offset = offset, source_resource = resource ) metl.manager.Manager( configparser.getTarget() ).run() sys.stdout.write( 'OK\n' ) sys.stdout.flush()
def write( filepath, records ): source = metl.source.staticsource.StaticSource( metl.fieldset.FieldSet( fields = [ metl.field.Field( 'key', metl.fieldtype.stringfieldtype.StringFieldType() ) ], fieldmap = metl.fieldmap.FieldMap({ 'key': 0 }) ) ) source.setResource( sourceRecords = [ [r] for r in records ] ) configparser = metl.configparser.ConfigParser( metl.config.Config( filepath ), init_on_start = False ) configparser.readers = [ source ] configparser.loadTarget() metl.manager.Manager( configparser.getTarget() ).run()
def metl_aggregate( argv = sys.argv ): parser = optparse.OptionParser( usage = 'Usage: %prog [options] CONFIG.YML FIELD' ) parser.add_option( "-p", "--path", dest = "path", default = None, help = "Conveyance of a folder, which is added to the PATH variable in order that the link in the YAML configuration could be run on an outside python file." ) parser.add_option( "-d", "--debug", action = 'store_true', default = False, help = "Debug mode, writes everything out as stdout." ) parser.add_option( "-l", "--limit", help = "Determines the number of elements for processing. An excellent option to test large files within small records until everything works as we would like to." ) parser.add_option( "-o", "--offset", default = 0, help = 'Determines from which element should the processing start.' ) parser.add_option( "-s", "--source", help = "If the configuration does not contain the route of the resource, it could be given here as well." ) (options, args) = parser.parse_args( argv[1:] ) if len( args ) != 2: parser.print_help() sys.exit() if options.path is not None: absdirectory = os.path.abspath( options.path ) sys.path.append( absdirectory ) configparser = metl.configparser.ConfigParser( metl.config.Config( args[0] ), debug = options.debug, limit = options.limit, offset = options.offset, source_resource = options.source ) target = metl.target.statictarget.StaticTarget( configparser.getTarget().getReader() ) target.setResource( silence = True ) target.initialize() target.write() target.finalize() values = set([]) for record in target.getResults(): values.add( record.getField( args[1] ).getValue() ) for value in values: print value
def main( argv = sys.argv ): parser = optparse.OptionParser( usage = 'Usage: %prog [options] CONFIG.YML' ) parser.add_option( "-t", "--targetMigration", dest = "target_migration_file", default = None, help = "During running, it prepares a migration file from the state of the present data." ) parser.add_option( "-m", "--migration", dest = "migration_file", default = None, help = 'Conveyance of previous migration file that was part of the previously run version.' ) parser.add_option( "-p", "--path", dest = "path", default = None, help = "Conveyance of a folder, which is added to the PATH variable in order that the link in the YAML configuration could be run on an outside python file." ) parser.add_option( "-d", "--debug", action = 'store_true', default = False, help = "Debug mode, writes everything out as stdout." ) parser.add_option( "-l", "--limit", help = "Determines the number of elements for processing." ) parser.add_option( "-o", "--offset", default = 0, help = 'Determines from which element should the processing start.' ) parser.add_option( "-s", "--source", help = "If the configuration does not contain the route of the resource, it could be given here as well." ) (options, args) = parser.parse_args( argv[1:] ) if len( args ) != 1: parser.print_help() sys.exit() if options.path is not None: absdirectory = os.path.abspath( options.path ) sys.path.append( absdirectory ) configparser = metl.configparser.ConfigParser( metl.config.Config( args[0] ), debug = options.debug, limit = options.limit, offset = options.offset, source_resource = options.source ) metl.manager.Manager( configparser.getTarget(), migration_resource = options.migration_file, target_migration_resource = options.target_migration_file ).run()
def metl_aggregate(argv=sys.argv): parser = optparse.OptionParser( usage='Usage: %prog [options] CONFIG.YML FIELD') parser.add_option( "-p", "--path", dest="path", default=None, help= "Conveyance of a folder, which is added to the PATH variable in order that the link in the YAML configuration could be run on an outside python file." ) parser.add_option("-d", "--debug", action='store_true', default=False, help="Debug mode, writes everything out as stdout.") parser.add_option( "-l", "--limit", help= "Determines the number of elements for processing. An excellent option to test large files within small records until everything works as we would like to." ) parser.add_option( "-o", "--offset", default=0, help='Determines from which element should the processing start.') parser.add_option( "-s", "--source", help= "If the configuration does not contain the route of the resource, it could be given here as well." ) (options, args) = parser.parse_args(argv[1:]) if len(args) != 2: parser.print_help() sys.exit() if options.path is not None: absdirectory = os.path.abspath(options.path) sys.path.append(absdirectory) configparser = metl.configparser.ConfigParser( metl.config.Config(args[0]), debug=options.debug, limit=options.limit, offset=options.offset, source_resource=options.source) target = metl.target.statictarget.StaticTarget( configparser.getTarget().getReader()) target.setResource(silence=True) target.initialize() target.write() target.finalize() values = set([]) for record in target.getResults(): values.add(record.getField(args[1]).getValue()) for value in values: print value
def main(argv=sys.argv): parser = optparse.OptionParser(usage='Usage: %prog [options] CONFIG.YML') parser.add_option( "-t", "--targetMigration", dest="target_migration_file", default=None, help= "During running, it prepares a migration file from the state of the present data." ) parser.add_option( "-m", "--migration", dest="migration_file", default=None, help= 'Conveyance of previous migration file that was part of the previously run version.' ) parser.add_option( "-p", "--path", dest="path", default=None, help= "Conveyance of a folder, which is added to the PATH variable in order that the link in the YAML configuration could be run on an outside python file." ) parser.add_option("-d", "--debug", action='store_true', default=False, help="Debug mode, writes everything out as stdout.") parser.add_option("-l", "--limit", help="Determines the number of elements for processing.") parser.add_option( "-o", "--offset", default=0, help='Determines from which element should the processing start.') parser.add_option( "-s", "--source", help= "If the configuration does not contain the route of the resource, it could be given here as well." ) (options, args) = parser.parse_args(argv[1:]) if len(args) != 1: parser.print_help() sys.exit() if options.path is not None: absdirectory = os.path.abspath(options.path) sys.path.append(absdirectory) configparser = metl.configparser.ConfigParser( metl.config.Config(args[0]), debug=options.debug, limit=options.limit, offset=options.offset, source_resource=options.source) metl.manager.Manager( configparser.getTarget(), migration_resource=options.migration_file, target_migration_resource=options.target_migration_file).run()