def _move_to_done(obj, eng):
     target_folder_full = get_storage_path(suffix=target_folder)
     collections = obj.data.get('collections', dict())
     for filename in collections.values():
         move(filename, join(target_folder_full, basename(filename)))
         obj.log.info("Moved {0} to {1}".format(
             filename,
             target_folder_full)
         )
 def _get_files_from_ftp(obj, eng):
     netrc_file = obj.extra_data["config"].get("ftp_netrc_file")
     target_folder_full = get_storage_path(suffix=target_folder)
     obj.data['all_files'], obj.data['new_files'] = ftp_download_files(
         source_folder,
         target_folder_full,
         server=obj.extra_data["config"]["ftp_server"],
         netrc_file=netrc_file
     )
     obj.log.info("{0} new files downloaded, in total {1} files".format(
         len(obj.data["new_files"]),
         len(obj.data["all_files"])
     ))
 def _unzip_files(obj, eng):
     target_folder_full = get_storage_path(suffix=target_folder)
     filenames = obj.data.get('all_files', list())
     all_extracted_files = []
     newly_extracted_files = []
     for filename in filenames:
         try:
             all_extracted, new_extracted = unzip(filename,
                                                  target_folder_full)
             all_extracted_files.extend(all_extracted)
             newly_extracted_files.extend(new_extracted)
         except BadZipfile as e:
             obj.log.error("Error unzipping file {0}: {1}".format(
                 filename,
                 e
             ))
             pass
     obj.data['all_extracted_files'] = all_extracted_files
     obj.data['newly_extracted_files'] = newly_extracted_files
     obj.log.debug("{0} new files extracted".format(
         len(obj.data["newly_extracted_files"])
     ))
    def _convert_files(obj, eng):
        from invenio_knowledge.api import get_kb_mappings
        mappings = dict(
            map(
                lambda item: (item['key'], item['value']),
                get_kb_mappings('JOURNALS')
            )
        )
        ws = WorldScientific(mappings)

        target_folder_full = get_storage_path(suffix=target_folder)

        args = obj.extra_data['args']

        # By default, we set the from date as today
        to_date = args.get("to_date") or datetime.now().strftime('%Y-%m-%d')

        # By last resort, we set the from date months before
        from_date = args.get("from_date")

        if not from_date:
            if args.get("reharvest"):
                # Since "beginning" of time when not specified
                from_date = datetime.strptime("1900-01-01", "%Y-%m-%d")
            else:
                # Dynamic date in the past when not specified and not reharvest
                from_date = datetime.now() - timedelta(weeks=weeks_threshold)\
                    .strftime('%Y-%m-%d')

        obj.extra_data['args']["to_date"] = to_date
        obj.extra_data['args']["from_date"] = from_date

        insert_files = []
        if args.get("reharvest"):
            filenames = obj.data['all_extracted_files']
        else:
            filenames = obj.data['newly_extracted_files']

        for filename in filenames:
            date = ws.get_date(filename)
            if date is None or (from_date <= date <= to_date):
                marc = ws.get_record(filename)
                if marc:
                    filename = basename(filename)
                    filename = join(target_folder_full, filename)
                    insert_files.append(filename)
                    with open(filename, 'w') as outfile:
                        outfile.write(marc)
            else:
                obj.log.info("Filtered out {0} ({1})".format(filename, date))

        obj.log.info("Converted {0}/{1} articles between {2} to {3}".format(
            len(insert_files),
            len(filenames),
            from_date,
            to_date
        ))

        obj.data['insert'] = insert_files
        obj.data["result_path"] = target_folder_full

        obj.log.debug("Saved converted files to {0}".format(target_folder_full))
        obj.log.debug("{0} files to add".format(
            len(obj.data["insert"]),
        ))