Exemple #1
0
def transfer_changed_datafile(data_file, hit_directory, log_directory = DirectoriesConfig().log_directory):
    """Transfers a changed file to XCDB by calling `transfer_file(data_file)`

    :param data_file: The soakdb that we want to check if it updated, :class:`TransferChangedDataFile` self.data_file
    :type data_file: str
    :param hit_directory: Directory to which the soakdb corresponds to. Usually :class:`TransferChangedDataFile` self.hit_directory
    :type hit_directory: str
    :return: Should return nothing but will update the
    :rtype: None
    """
    print(data_file)
    maint_exists = db_functions.check_table_sqlite(data_file, 'mainTable')

    if maint_exists == 1:
        soakdb_query = SoakdbFiles.objects.get(filename=data_file)
        print(soakdb_query)
        split_path = data_file.split('database')
        search_path = split_path[0]

        # remove pandda data transfer done file
        if os.path.isfile(os.path.join(search_path, 'transfer_pandda_data.done')):
            os.remove(os.path.join(search_path, 'transfer_pandda_data.done'))

        log_files = find_log_files(search_path).rsplit()
        print(log_files)
        for log in log_files:
            print(f"{log}.run.done")
            if os.path.isfile(f"{log}.run.done"):
                os.remove(f"{log}.run.done")
            if os.path.isfile(f"{log}.sites.done"):
                os.remove(f"{log}.sites.done")
            if os.path.isfile(f"{log}.events.done"):
                os.remove(f"{log}.events.done")

        # find_logs_out_files = glob.glob(str(search_path + '*.txt'))
        find_logs_out_files = glob.glob(f"{search_path}*.txt")

        for f in find_logs_out_files:
            if is_date(f.replace(search_path, '').replace('.txt', '')):
                os.remove(f)

        out, err, proposal = db_functions.pop_soakdb(data_file)
        db_functions.pop_proposals(proposal)

    else:
        print('Main Table does not exist!')

    transfer_file(data_file)
Exemple #2
0
def run_transfer(filename):
    status_query = SoakdbFiles.objects.get(filename=filename).status
    print('status query: ' + str(status_query) + '(0=new, 1=updated)')
    # if it is a changed file - do the delete things
    if status_query == 1:
        # maint_exists = db_functions.check_table_sqlite(filename, 'mainTable')
        #
        # if maint_exists == 1:
        soakdb_query = SoakdbFiles.objects.get(filename=filename)
        print('Updating/adding a new entry for file... (not deleting)')
        #soakdb_query.delete()

    # the next step is always the same
    out, err, proposal = db_functions.pop_soakdb(filename)
    db_functions.pop_proposals(proposal)

    transfer_file(filename)
Exemple #3
0
    def run(self):
        # delete all fields from soakdb filename
        maint_exists = db_functions.check_table_sqlite(self.data_file,
                                                       'mainTable')
        if maint_exists == 1:
            soakdb_query = SoakdbFiles.objects.get(filename=self.data_file)

            crystals = Crystal.objects.filter(visit=soakdb_query)

            for crystal in crystals:
                refinement = Refinement.objects.get(crystal_name=crystal)

                target_name = str(crystal.target.target_name).upper()
                crystal_name = str(crystal.crystal_name)
                proasis_crystal_directory = os.path.join(
                    self.hit_directory, target_name, crystal_name)

                if ProasisHits.objects.filter(crystal_name=crystal).exists():
                    proasis_hit = ProasisHits.objects.filter(
                        crystal_name=crystal)
                    for hit in proasis_hit:
                        for path in glob.glob(
                                os.path.join(
                                    os.getcwd(), 'logs/proasis/hits',
                                    str(hit.crystal_name.crystal_name + '_' +
                                        hit.modification_date + '*'))):
                            os.remove(path)
                        if os.path.isdir(proasis_crystal_directory):
                            shutil.rmtree(
                                os.path.join(proasis_crystal_directory))

                        if ProasisOut.objects.filter(proasis=hit).exists:
                            for obj in ProasisOut.objects.filter(proasis=hit):
                                obj.delete()
                        hit.delete()

            soakdb_query.delete()

            out, err, proposal = db_functions.pop_soakdb(self.data_file)
            db_functions.pop_proposals(proposal)

        transfer_file(self.data_file)

        with self.output().open('w') as f:
            f.write('')
Exemple #4
0
def transfer_all_fed_ids_and_datafiles(soak_db_filelist):
    """Transfers fedids and datafiles from soakdb filepaths to XCDB, used to populate the proposals table

    :param soak_db_filelist: Soakdb filepath as defined within :class:`transfer_soakdb.TransferAllFedIDsAndDatafiles` as self.input()
    :type soak_db_filelist: str
    :return: Should return nothing, but will populate the proposals table in XCDB.
    :rtype: None
    """
    print(soak_db_filelist)
    with open(soak_db_filelist, 'rb') as database_list:
        for database_file in database_list.readlines():
            database_file = database_file.replace('\n', '')

            out, err, proposal = db_functions.pop_soakdb(database_file)
            print(out)
            print(err)
            print(proposal)

    proposal_list = list(SoakdbFiles.objects.values_list('proposal', flat=True))

    for proposal_number in set(proposal_list):
        db_functions.pop_proposals(proposal_number)
Exemple #5
0
    def run(self):

        # use list from previous step as input to write to postgres
        with self.input().open('r') as database_list:
            for database_file in database_list.readlines():
                database_file = database_file.replace('\n', '')

                # populate the soakdb table for each db file found by FindSoakDBFiles
                out, err, proposal = db_functions.pop_soakdb(database_file)
                print(out)
                print(err)
                print(proposal)

        # return a list of all proposals from db
        proposal_list = list(SoakdbFiles.objects.values_list('proposal', flat=True))

        # add fedid permissions via proposals table
        for proposal_number in set(proposal_list):
            db_functions.pop_proposals(proposal_number)

        # write output to show job done
        with self.output().open('w') as f:
            f.write('TransferFeDIDs DONE')
Exemple #6
0
    def run(self):
        # connect to central postgres db
        conn, c = db_functions.connectDB()

        # use list from previous step as input to write to postgres
        with self.input().open('r') as database_list:
            for database_file in database_list.readlines():
                database_file = database_file.replace('\n', '')

                out, err, proposal = db_functions.pop_soakdb(database_file)

        proposal_list = []
        c.execute('SELECT proposal FROM soakdb_files')
        rows = c.fetchall()
        for row in rows:
            proposal_list.append(str(row[0]))

        for proposal_number in set(proposal_list):
            db_functions.pop_proposals(proposal_number)

        c.close()

        with self.output().open('w') as f:
            f.write('TransferFeDIDs DONE')
Exemple #7
0
    def run(self):
        # a list to hold filenames that have been checked
        checked = []

        # Status codes:-
        # 0 = new
        # 1 = changed
        # 2 = not changed

        print('INPUT NAME:')
        print(self.input()[1].path)

        with open(self.input()[1].path, 'r') as f:
            files = f.readlines()
            print('FILES:')
            print(files)

        for filename in files:
            # remove any newline characters
            filename_clean = filename.rstrip('\n')
            # find the relevant entry in the soakdbfiles table

            soakdb_query = list(SoakdbFiles.objects.filter(filename=filename_clean))

            print(len(soakdb_query))

            # raise an exception if the file is not in the soakdb table
            if len(soakdb_query) == 0:
                print('LEN=0')
                out, err, prop = db_functions.pop_soakdb(filename_clean)
                db_functions.pop_proposals(prop)

            # only one entry should exist per file
            if len(soakdb_query) == 1:
                print('LEN=1')
                # get the filename back from the query
                data_file = soakdb_query[0].filename
                # add the file to the list of those that have been checked
                checked.append(data_file)
                # get the modification date as stored in the db
                old_mod_date = soakdb_query[0].modification_date
                # get the current modification date of the file
                current_mod_date = misc_functions.get_mod_date(data_file)
                # get the id of the entry to write to
                id_number = soakdb_query[0].id

                print(old_mod_date)
                if not old_mod_date:
                    soakdb_query[0].modification_date = current_mod_date
                    soakdb_query[0].save()
                    old_mod_date = 0
                print(current_mod_date)

                # if the file has changed since the db was last updated for the entry, change status to indicate this
                try:
                    if int(current_mod_date) > int(old_mod_date):
                        update_status = SoakdbFiles.objects.get(id=id_number)
                        update_status.status = 1
                        update_status.save()
                except ValueError:
                    raise Exception(str('current_mod_date: ' + str(current_mod_date)
                                        + ', old_mod_date: ' + str(old_mod_date)))

            # if there is more than one entry, raise an exception (should never happen - filename field is unique)
            if len(soakdb_query) > 1:
                raise Exception('More than one entry for file! Something has gone wrong!')

            # if the file is not in the database at all
            if filename_clean not in checked:
                # add the file to soakdb
                out, err, proposal = db_functions.pop_soakdb(filename_clean)
                # add the proposal to proposal
                db_functions.pop_proposals(proposal)
                # retrieve the new db entry
                soakdb_query = list(SoakdbFiles.objects.filter(filename=filename_clean))
                # get the id to update
                id_number = soakdb_query[0].id
                # update the relevant status to 0, indicating it as a new file
                update_status = SoakdbFiles.objects.get(id=id_number)
                update_status.status = 0
                update_status.save()

        # if the lab table is empty, no data has been transferred from the datafiles, so set status of everything to 0
        lab = list(Lab.objects.all())
        if not lab:
            # this is to set all file statuses to 0 (new file)
            soakdb = SoakdbFiles.objects.all()
            for filename in soakdb:
                filename.status = 0
                filename.save()

        # write output to signify job done
        with self.output().open('w') as f:
            f.write('')
Exemple #8
0
    def run(self):
        print(self.data_file)
        # delete all fields from soakdb filename
        maint_exists = db_functions.check_table_sqlite(self.data_file, 'mainTable')

        if maint_exists == 1:
            soakdb_query = SoakdbFiles.objects.get(filename=self.data_file)
            print(soakdb_query)
            # for pandda file finding
            split_path = self.data_file.split('database')
            search_path = split_path[0]

            # remove pandda data transfer done file
            if os.path.isfile(os.path.join(search_path, 'transfer_pandda_data.done')):
                os.remove(os.path.join(search_path, 'transfer_pandda_data.done'))

            log_files = find_log_files(search_path).rsplit()
            print(log_files)

            for log in log_files:
                print(str(log + '.run.done'))
                if os.path.isfile(str(log + '.run.done')):
                    os.remove(str(log + '.run.done'))
                if os.path.isfile(str(log + '.sites.done')):
                    os.remove(str(log + '.sites.done'))
                if os.path.isfile(str(log + '.events.done')):
                    os.remove(str(log + '.events.done'))

            find_logs_out_files = glob.glob(str(search_path + '*.txt'))

            for f in find_logs_out_files:
                if is_date(f.replace(search_path, '').replace('.txt', '')):
                    os.remove(f)

            crystals = Crystal.objects.filter(visit=soakdb_query)

            for crystal in crystals:
                target_name = str(crystal.target.target_name).upper()
                crystal_name = str(crystal.crystal_name)
                proasis_crystal_directory = os.path.join(self.hit_directory, target_name.upper(), crystal_name)

                if ProasisHits.objects.filter(crystal_name=crystal).exists():
                    proasis_hit = ProasisHits.objects.filter(crystal_name=crystal)
                    for hit in proasis_hit:
                        for path in glob.glob(os.path.join(DirectoriesConfig().log_directory, 'proasis/hits',
                                                           str(hit.crystal_name.crystal_name +
                                                               '_' + hit.modification_date + '*'))):
                            os.remove(path)
                        if os.path.isdir(proasis_crystal_directory):
                            shutil.rmtree(os.path.join(proasis_crystal_directory), ignore_errors=True)

                        if ProasisOut.objects.filter(proasis=hit).exists:
                            for obj in ProasisOut.objects.filter(proasis=hit):
                                if obj.root:
                                    delete_files = ['verne.transferred', 'PROPOSALS', 'VISITS', 'visits_proposals.done']
                                    for f in delete_files:
                                        if os.path.isfile(os.path.join(obj.root, '/'.join(obj.start.split('/')[:-2]),
                                                                       f)):
                                            os.remove(os.path.join(obj.root, '/'.join(obj.start.split('/')[:-2]), f))
                                    shutil.rmtree(os.path.join(obj.root, obj.start))
                                obj.delete()
                        hit.delete()

            soakdb_query.delete()

            out, err, proposal = db_functions.pop_soakdb(self.data_file)
            db_functions.pop_proposals(proposal)

        else:
            print('MAIN TABLE DOES NOT EXIST!')

        transfer_file(self.data_file)

        with self.output().open('w') as f:
            f.write('')
Exemple #9
0
def check_files(soak_db_filepath):
    """Check if soakdb file has been updated since last run

    :param soak_db_filepath: Soakdb filepath as defined
    within :class:`transfer_soakdb.CheckFiles` as self.input()[1].path
    :type soak_db_filepath: str
    :return: Will return nothing but should update the status of soakdb file if it needs to...
    :rtype: None
    """
    # Beginning of run(self)
    checked = []
    # Status codes:-
    # 0 = new
    # 1 = changed
    # 2 = not changed

    # self.input()[1].path = soak_db_filepath?
    print(f'INPUT NAME: {soak_db_filepath}')

    # Open file
    with open(soak_db_filepath, 'r') as f:
        files = f.readlines()
        print(f'FILES: {files}')

    for filename in files:
        filename_clean = filename.rstrip('\n')
        soakdb_query = list(SoakdbFiles.objects.filter(filename=filename_clean))
        print(len(soakdb_query))

        # Consider Switch instead of IFs?
        if len(soakdb_query) == 0:
            print('LEN=0')
            out, err, prop = db_functions.pop_soakdb(filename_clean)
            db_functions.pop_proposals(prop)

        if len(soakdb_query) == 1:
            print('LEN=1')
            # Get filename from query
            data_file = soakdb_query[0].filename
            # add file to list which have been checked
            checked.append(data_file)
            # Get last modification date as stored in soakdb
            old_mod_date = soakdb_query[0].modification_date
            # Get current modification date of file
            current_mod_date = misc_functions.get_mod_date(data_file)
            # get the id of entry to write to
            id_number = soakdb_query[0].id

            print(old_mod_date)
            if not old_mod_date:
                soakdb_query[0].modification_date = current_mod_date
                soakdb_query[0].save()
                old_mod_date = 0

            print(current_mod_date)

            # if the file has changed since the db was last updated for the entry, change status to indicate this
            try:
                if int(current_mod_date) > int(old_mod_date):
                    update_status = SoakdbFiles.objects.get(id=id_number)
                    update_status.status = 1
                    update_status.save()
            except ValueError:
                raise Exception(f"current_mod_date: {current_mod_date}, old_mod_date: {old_mod_date}")

        if len(soakdb_query) > 1:
            raise Exception('More than one entry for file! Something has gone wrong!')

        # If file isn't in XCDB
        if filename_clean not in checked:
            # Add to soakdb
            out, err, proposal = db_functions.pop_soakdb(filename_clean)
            db_functions.pop_proposals(proposal)
            soakdb_query = list(SoakdbFiles.objects.filter(filename=filename_clean))
            id_number = soakdb_query[0].id
            update_status = SoakdbFiles.objects.get(id=id_number)
            update_status.status = 0
            update_status.save()

    lab = list(Lab.objects.all())
    if not lab:
        # Set all file statuses to 0
        soak_db = SoakdbFiles.objects.all()
        for filename in soak_db:
            filename.status = 0
            filename.save()
Exemple #10
0
    def run(self):

        conn, c = db_functions.connectDB()
        exists = db_functions.table_exists(c, 'soakdb_files')

        checked = []

        # Status codes:-
        # 0 = new
        # 1 = changed
        # 2 = not changed

        if exists:
            with self.input().open('r') as f:
                files = f.readlines()

            for filename in files:

                filename_clean = filename.rstrip('\n')

                c.execute(
                    'select filename, modification_date, status_code from soakdb_files where filename like %s;',
                    (filename_clean, ))

                for row in c.fetchall():
                    if len(row) > 0:
                        data_file = str(row[0])
                        checked.append(data_file)
                        old_mod_date = str(row[1])
                        current_mod_date = misc_functions.get_mod_date(
                            data_file)

                        if current_mod_date > old_mod_date:
                            c.execute(
                                'UPDATE soakdb_files SET status_code = 1 where filename like %s;',
                                (filename_clean, ))
                            c.execute(
                                'UPDATE soakdb_files SET modification_date = %s where filename like %s;',
                                (current_mod_date, filename_clean))
                            conn.commit()

                if filename_clean not in checked:
                    out, err, proposal = db_functions.pop_soakdb(
                        filename_clean)
                    db_functions.pop_proposals(proposal)
                    c.execute(
                        'UPDATE soakdb_files SET status_code = 0 where filename like %s;',
                        (filename_clean, ))
                    conn.commit()

            c.execute('select filename from soakdb_files;')

            # for row in c.fetchall():
            #     if str(row[0]) not in checked:
            #         data_file = str(row[0])

        exists = db_functions.table_exists(c, 'lab')
        if not exists:
            c.execute('UPDATE soakdb_files SET status_code = 0;')
            conn.commit()

        with self.output().open('w') as f:
            f.write('')
Exemple #11
0
def check_file(filename):
    status=2
    # remove any newline characters
    filename_clean = filename.rstrip('\n')
    # find the relevant entry in the soakdbfiles table

    soakdb_query = list(SoakdbFiles.objects.filter(filename=filename_clean))

    # raise an exception if the file is not in the soakdb table - not necessary here, I think
    # if len(soakdb_query) == 0:
    #     print('LEN=0')
    #     out, err, prop = db_functions.pop_soakdb(filename_clean)
    #     db_functions.pop_proposals(prop)

    # only one entry should exist per file
    if len(soakdb_query) == 1:
        # get the filename back from the query
        data_file = soakdb_query[0].filename
        # add the file to the list of those that have been checked
        # checked.append(data_file)
        # get the modification date as stored in the db
        old_mod_date = soakdb_query[0].modification_date
        # get the current modification date of the file
        current_mod_date = misc_functions.get_mod_date(data_file)
        # get the id of the entry to write to
        id_number = soakdb_query[0].id

        if not old_mod_date:
            soakdb_query[0].modification_date = current_mod_date
            soakdb_query[0].save()
            old_mod_date = 0

        # if the file has changed since the db was last updated for the entry, change status to indicate this
        try:
            if int(current_mod_date) > int(old_mod_date):
                update_status = SoakdbFiles.objects.get(id=id_number)
                update_status.status = 1
                status = 1
                update_status.save()
        except ValueError:
            raise Exception(str('current_mod_date: ' + str(current_mod_date)
                                + ', old_mod_date: ' + str(old_mod_date)))

        print(current_mod_date)
        print(old_mod_date)

    # if there is more than one entry, raise an exception (should never happen - filename field is unique)
    if len(soakdb_query) > 1:
        raise Exception('More than one entry for file! Something has gone wrong!')

    # if the file is not in the database at all
    if len(soakdb_query) == 0:
        print('This is a new soakDB file, just setting it up in the database!')
        # add the file to soakdb
        out, err, proposal = db_functions.pop_soakdb(filename_clean)
        # add the proposal to proposal
        db_functions.pop_proposals(proposal)
        # retrieve the new db entry
        soakdb_query = list(SoakdbFiles.objects.filter(filename=filename_clean))
        # get the id to update
        id_number = soakdb_query[0].id
        # update the relevant status to 0, indicating it as a new file
        update_status = SoakdbFiles.objects.get(id=id_number)
        update_status.status = 0
        update_status.save()
        status = 0

    # else:
    #     print('The file has not been updated, using existing XCDB data...')
    #     status = 2

    # if the lab table is empty, no data has been transferred from the datafiles, so set status of everything to 0


    lab = list(Lab.objects.all())
    if not lab:
        # this is to set all file statuses to 0 (new file)
        soakdb = SoakdbFiles.objects.all()
        for filename in soakdb:
            filename.status = 0
            filename.save()

    return status