def getModuleInfoFileObjects(self):
fs = FileSystemAdapter()
fs.chdir(self.path)
for filepath in fs.glob('*.info'):
module = os.path.basename(filepath).split('.')[
0] # assume dot is not allowed for module filenames
yield ModuleInfoFile(filepath, module, self.machine_name,
self.major_version, self.is_core_project)
class ProjectsLinkedDropSelfLinkedPipeline(object):
fs = FileSystemAdapter(settings.PARSLEY_CONTRIBUTED_PROJECTS_ROOT)
def process_item(self, current_item, current_spider):
if current_spider.name == "projects_linked" and current_item[
'linked_from'] == current_item['linked_to']:
raise DropItem("Self-linked item")
return current_item
def parse(self, response):
"""
@url http://localhost/
@returns items 33 33
@returns requests 0 0
"""
fs = FileSystemAdapter(PARSLEY_CORE_PROJECTS_ROOT)
projects = fs.get_subdirectories_names()
for project in projects:
data = dict()
data['machine_name'] = project
data['project_uid'] = 0
data['project_maintenance_status'] = 'Core module'
data['project_development_status'] = 'Core module'
item = ScrapyItemListWrapper(data.keys())
for key, value in data.iteritems():
item[key] = value
yield item
class ProjectsContributedGitClonePipeline(object):
fs = FileSystemAdapter(settings.PARSLEY_CONTRIBUTED_PROJECTS_ROOT)
def process_item(self, current_item, current_spider):
if current_spider.name == "projects_releases":
project_machine_name = current_item['release_project_machine_name']
self._git_clone(project_machine_name)
return current_item
def _git_clone(self, project_machine_name): # todo
# self.fs.run("git clone --no-checkout http://git.drupal.org/project/%s.git" % project_machine_name)
pass
def getModuleInfoFileObjects(self):
fs = FileSystemAdapter()
fs.chdir(self.path)
for filepath in fs.glob('*.info'):
module = os.path.basename(filepath).split('.')[0] # assume dot is not allowed for module filenames
yield ModuleInfoFile(filepath, module, self.machine_name, self.major_version, self.is_core_project)