def parse_protein(project_id, bucket_name, filename, outfilename, metadata):
    """Download and convert blob into dataframe
       Transform the file: includes data cleaning
       Add Metadata information
    """
    # setup logging
    configure_logging('protein', "logs/" + metadata['AliquotBarcode'] + '.log')

    # connect to the cloud bucket
    gcs = GcsConnector(project_id, bucket_name)

    #main steps: download, convert to df, cleanup, transform, add metadata
    data_df = gcutils.convert_blob_to_dataframe(gcs,
                                                project_id,
                                                bucket_name,
                                                filename,
                                                skiprows=1)
    data_df = additional_changes(data_df)
    data_df = add_metadata(data_df, metadata)

    # validation
    tests.assert_notnull_property(data_df, columns_list=['Protein_Name'])

    # upload the contents of the dataframe in njson format
    status = gcs.convert_df_to_njson_and_upload(data_df,
                                                outfilename,
                                                metadata=metadata)
    return status
def parse_protein(project_id, bucket_name, filename, outfilename, metadata):
    """Download and convert blob into dataframe
       Transform the file: includes data cleaning
       Add Metadata information
    """
    # setup logging
    configure_logging("protein", "logs/" + metadata["AliquotBarcode"] + ".log")

    # connect to the cloud bucket
    gcs = GcsConnector(project_id, bucket_name)

    # main steps: download, convert to df, cleanup, transform, add metadata
    data_df = gcutils.convert_blob_to_dataframe(gcs, project_id, bucket_name, filename, skiprows=1)
    data_df = additional_changes(data_df)
    data_df = add_metadata(data_df, metadata)

    # validation
    tests.assert_notnull_property(data_df, columns_list=["Protein_Name"])

    # upload the contents of the dataframe in njson format
    status = gcs.convert_df_to_njson_and_upload(data_df, outfilename, metadata=metadata)
    return status
def parse_protein(project_id, bucket_name, filename, outfilename, metadata):
    """Download and convert blob into dataframe
       Transform the file: includes data cleaning
       Add Metadata information
    """
    # setup logging
    log = configure_logging(
        'protein',
        "logs/protein_transform_" + metadata['AliquotBarcode'] + '.log')
    try:
        log.info('start transform of %s' % (metadata['AliquotBarcode']))
        # connect to the cloud bucket
        gcs = GcsConnector(project_id, bucket_name)

        #main steps: download, convert to df, cleanup, transform, add metadata
        data_df = gcutils.convert_blob_to_dataframe(gcs,
                                                    project_id,
                                                    bucket_name,
                                                    filename,
                                                    skiprows=1,
                                                    log=log)
        log.info('\tadd changes and metadata for %s' %
                 (metadata['AliquotBarcode']))
        data_df = additional_changes(data_df)
        data_df = add_metadata(data_df, metadata)

        # validation
        tests.assert_notnull_property(data_df, columns_list=['Protein_Name'])

        # upload the contents of the dataframe in njson format
        status = gcs.convert_df_to_njson_and_upload(data_df,
                                                    outfilename,
                                                    metadata=metadata)
        log.info('finished transform of %s' % (metadata['AliquotBarcode']))
    except Exception as e:
        log.exception('problem transforming %s' % (metadata['AliquotBarcode']))
        raise e
    return status
def main(datatype, config_file, max_workers, dry_run, create_new, debug):
    """
    Pipeline
    """
    # config params
    config = json.load(open(config_file))
    project_id = config['project_id']
    bucket_name = config['buckets']['open']
    table_task_queue = 'task_queue'
    table_task_queue_status = 'task_queue_status' 
    db_filename = 'etl-{0}.db'.format(datatype)
    log_filename = 'etl_{0}.log'.format(datatype)
    log_name = 'etl_{0}'.format(datatype)

    log.info('start pipeline for %s' % (datatype))
    # check if the table exists and issue warning
    if os.path.exists(db_filename):
       log.warning('Using the already available database file - {0}'.format(db_filename))
       time.sleep(2)
    
    # connect to the database
    conn = sqlite3.connect(db_filename, check_same_thread=False)
   
    #-------------------------------
    # Submit to task queue
    #-------------------------------
    print "="*30 + "\nQuerying Google Cloud SQL metadata_data table"
    queue_df = extract_functions[datatype](config)
    submit_to_queue(queue_df, conn, table_task_queue)

    #--------------
    # Tests
    #--------------
    tests.assert_notnull_property(queue_df, columns_list=['SampleTypeCode', 'SampleTypeLetterCode',\
                     'Study', 'Platform', 'SampleBarcode', 'OutDatafileNameKey',\
                                         'ParticipantBarcode', 'DatafileNameKey', 'AliquotBarcode'])

    if create_new:
         # delete the old queue(task_queue_status) and re-run
        conn.execute('DROP TABLE IF EXISTS {0}'.format(table_task_queue_status))

    # Validate and get diff ; restart ETL; also takes care of errors
    queue_df = validate_and_get_diff(conn, queue_df, table_task_queue_status)

    if debug:
        # debug mode runs top 30 rows
        log.debug('Running in debug mode (first 30 records)')
        queue_df = queue_df.head(30)

    if dry_run:
        log.info('finished dry run for %s' % (datatype))
        sys.exit()


    #--------------------------------------------
    # Execution
    #------------------------------------------------------
    pmr = process_manager.ProcessManager(max_workers=max_workers, db=db_filename, table=table_task_queue_status, log=log)
    for index, row in queue_df.iterrows():
        metadata = row.to_dict()
        inputfilename = metadata['DatafileNameKey']
        outputfilename = metadata['OutDatafileNameKey']
        # transform
        #transform_functions[datatype]( project_id, bucket_name,\
        #                inputfilename, outputfilename, metadata)
        future = pmr.submit(transform_functions[datatype], project_id, bucket_name,\
                        inputfilename, outputfilename, metadata)
    
        time.sleep(0.1 + 0.5  * random.random())
        if index % 100 == 0:
            time.sleep(5)

    pmr.start()
    log.info('finished pipeline for %s' % (datatype))
def main(datatype, config_file, max_workers, dry_run, create_new, debug):
    """
    Pipeline
    """
    # config params
    config = json.load(open(config_file))
    project_id = config['project_id']
    bucket_name = config['buckets']['open']
    table_task_queue = 'task_queue'
    table_task_queue_status = 'task_queue_status' 
    db_filename = 'etl-{0}.db'.format(datatype)
    log_filename = 'etl_{0}.log'.format(datatype)
    log_name = 'etl_{0}'.format(datatype)

    # check if the table exists and issue warning
    if os.path.exists(db_filename):
       log.warning('Using the already available database file - {0}'.format(db_filename))
       time.sleep(2)
    
    # connect to the database
    conn = sqlite3.connect(db_filename, check_same_thread=False)
   
    #-------------------------------
    # Submit to task queue
    #-------------------------------
    print "="*30 + "\nQuerying Google Cloud SQL metadata_data table"
    queue_df = extract_functions[datatype](config)
    submit_to_queue(queue_df, conn, table_task_queue)

    #--------------
    # Tests
    #--------------
    tests.assert_notnull_property(queue_df, columns_list=['SampleTypeCode', 'SampleTypeLetterCode',\
                     'Study', 'Platform', 'SampleBarcode', 'OutDatafileNameKey',\
                                         'ParticipantBarcode', 'DatafileNameKey', 'AliquotBarcode'])

    if create_new:
         # delete the old queue(task_queue_status) and re-run
        conn.execute('DROP TABLE IF EXISTS {0}'.format(table_task_queue_status))

    # Validate and get diff ; restart ETL; also takes care of errors
    queue_df = validate_and_get_diff(conn, queue_df, table_task_queue_status)

    if debug:
        # debug mode runs top 30 rows
        log.debug('Running in debug mode (first 30 records)')
        queue_df = queue_df.head(30)

    if dry_run:
        sys.exit()


    #--------------------------------------------
    # Execution
    #------------------------------------------------------
    pmr = process_manager.ProcessManager(max_workers=max_workers, db=db_filename, table=table_task_queue_status)
    for index, row in queue_df.iterrows():
        metadata = row.to_dict()
        inputfilename = metadata['DatafileNameKey']
        outputfilename = metadata['OutDatafileNameKey']
        # transform
        #transform_functions[datatype]( project_id, bucket_name,\
        #                inputfilename, outputfilename, metadata)
        future = pmr.submit(transform_functions[datatype], project_id, bucket_name,\
                        inputfilename, outputfilename, metadata)
    
        time.sleep(0.1 + 0.5  * random.random())
        if index % 100 == 0:
            time.sleep(5)

    pmr.start()